fix: don't abort the mirror run when a dependency download fails

morgan/__init__.py

@@ -111,10 +111,17 @@ def mirror(self, requirement_string: str):
         while len(deps) > 0:
             next_deps = {}
             for dep in deps:
-                more_deps = self._mirror(
-                    deps[dep]["requirement"],
-                    required_by=deps[dep]["required_by"],
-                )
+                try:
+                    more_deps = self._mirror(
+                        deps[dep]["requirement"],
+                        required_by=deps[dep]["required_by"],
+                    )
+                except urllib.error.HTTPError as err:
+                    # dependency names come from package metadata and may not exist
+                    # on the index (404). Skip them so one broken dependency does
+                    # not abort the entire mirror run.
+                    print(f"\tError: {err}")
+                    continue
                 if more_deps:
                     next_deps.update(more_deps)
             deps = next_deps.copy()

tests/test_init.py

@@ -3,6 +3,7 @@
 
 import argparse
 import contextlib
+import email.message
 import hashlib
 import os
 import urllib.error
@@ -569,3 +570,69 @@ def test_filter_files_selects_latest_compatible_version_of_sdist_only(
         assert {
             "sample_package-1.0.0.tar.gz",
         } == set(filenames), f"Wrong packages selected. Got: {filenames}"
+
+
+class TestMirrorDependencyErrors:
+    @pytest.fixture
+    def mirrorer(self, tmp_path):
+        config_path = os.path.join(tmp_path, "morgan.ini")
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(
+                """
+                [env.test_env]
+                python_version = 3.10
+                sys_platform = linux
+                platform_machine = x86_64
+
+                [requirements]
+                toplevel = >=1.0
+                """,
+            )
+        args = argparse.Namespace(
+            index_path=str(tmp_path),
+            index_url=PYPI_ADDRESS,
+            config=config_path,
+            mirror_all_versions=False,
+            package_type_regex=r"(whl|zip|tar\.gz)",
+            mirror_all_wheels=False,
+        )
+        return Mirrorer(args)
+
+    def test_mirror_continues_when_dependency_fails(self, mirrorer, monkeypatch):
+        """A 404 on one dependency must not abort the remaining dependencies."""
+        toplevel = parse_requirement("toplevel>=1.0")
+        processed = []
+
+        def fake_mirror(requirement, required_by=None):
+            processed.append(requirement.name)
+            if required_by is None:
+                # inject one failing and one working dependency; the failing
+                # one comes first so an uncaught error would skip the other
+                return {
+                    "missing-dep": {
+                        "requirement": parse_requirement("missing-dep"),
+                        "required_by": toplevel,
+                    },
+                    "present-dep": {
+                        "requirement": parse_requirement("present-dep"),
+                        "required_by": toplevel,
+                    },
+                }
+            if requirement.name == "missing-dep":
+                raise urllib.error.HTTPError(
+                    url="https://example.com/missing-dep/",
+                    code=404,
+                    msg="Not Found",
+                    hdrs=email.message.Message(),
+                    fp=None,
+                )
+            return None
+
+        # pylint: disable=W0212
+        monkeypatch.setattr(mirrorer, "_mirror", fake_mirror)
+
+        mirrorer.mirror("toplevel>=1.0")
+
+        assert "present-dep" in processed, (
+            "Dependencies after a failing one should still be processed"
+        )