Determine hybrid_format before filtering architectures

pulp_deb/app/tasks/synchronizing.py

@@ -569,6 +569,22 @@ async def _handle_distribution(self, distribution):
         stored_distribution = "flat-repo" if is_flat else distribution
 
         log.info(_('Downloading Release file for distribution: "{}"').format(distribution))
+
+
+        # Retrieve and interpret any 'No-Support-for-Architecture-all' value:
+        # We will refer to the presence of 'No-Support-for-Architecture-all: Packages' in a Release
+        # file as indicating "hybrid format". For more info, see:
+        # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all
+        no_support_for_arch_all = release_file_dict.get("No-Support-for-Architecture-all", "")
+        if no_support_for_arch_all.strip() == "Packages":
+            hybrid_format = True
+        elif not no_support_for_arch_all:
+            hybrid_format = False
+        else:
+            raise UnknownNoSupportForArchitectureAllValue(
+                release_file.relative_path, no_support_for_arch_all
+            )
+
         # Create release_file
         if is_flat:
             upstream_file_dir = distribution.strip("/")
@@ -635,6 +651,14 @@ async def _handle_distribution(self, distribution):
             architectures = filter_arch_tokens(
                 release_file.architectures, self.remote.architectures, distribution
             )
+            # For hybrid-format repos, remove "all" from the component processing list
+            # (it will be handled separately in _handle_component)
+            if hybrid_format:
+                architectures = [
+                    (base_arch, pub_arch, variant)
+                    for (base_arch, pub_arch, variant) in architectures
+                    if base_arch != "all"
+                ]
 
         for base_arch, published_arch, variant in architectures:
             release_architecture_dc = DeclarativeContent(
@@ -647,20 +671,6 @@ async def _handle_distribution(self, distribution):
             )
             await self.put(release_architecture_dc)
 
-        # Retrieve and interpret any 'No-Support-for-Architecture-all' value:
-        # We will refer to the presence of 'No-Support-for-Architecture-all: Packages' in a Release
-        # file as indicating "hybrid format". For more info, see:
-        # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all
-        no_support_for_arch_all = release_file_dict.get("No-Support-for-Architecture-all", "")
-        if no_support_for_arch_all.strip() == "Packages":
-            hybrid_format = True
-        elif not no_support_for_arch_all:
-            hybrid_format = False
-        else:
-            raise UnknownNoSupportForArchitectureAllValue(
-                release_file.relative_path, no_support_for_arch_all
-            )
-
         # collect file references in new dict
         file_references = defaultdict(deb822.Deb822Dict)
         for digest_name in ["SHA512", "SHA256", "SHA1", "MD5sum"]:
@@ -738,18 +748,6 @@ async def _handle_component(
                     [x for x in release_file.architectures.split() if x != "all"]
                 )
 
-        # Putting this here because it fixes an issue where debian packages with the parameter
-        # architectures='all' are missing after sync/publish. It is not really clear why and
-        # needs investigation once there are tests for this issue. Best guess it hasis something do
-        # with the asynchronous handling of the tasks and removing something from a dict without
-        # a copy.
-        if hybrid_format:
-            architectures = [
-                (base_arch, pub_arch, variant)
-                for (base_arch, pub_arch, variant) in architectures
-                if base_arch != "all"
-            ]
-
         pending_tasks = []
         # Handle package indices
         for base_arch, index_arch, variant in architectures:

pulp_deb/tests/functional/api/test_publish.py

@@ -114,6 +114,7 @@ def _verify_publication_data(publication, expected, is_nested=False):
         )
         release_file_path = os.path.join(expected[release_file_folder], release_type)
 
+        assert any(pi.relative_path.endswith("asgard/binary-all/Packages") for pi in package_indices)
         assert release_file_path == release_file.relative_path
         assert release_file.distribution == release.distribution == expected["distribution"]
         assert release_file.codename == release.codename == expected["codename"]

pulp_deb/tests/functional/api/test_sync.py

@@ -17,10 +17,12 @@
     DEB_FIXTURE_SUMMARY,
     DEB_FIXTURE_UPDATE_REPOSITORY_NAME,
     DEB_FIXTURE_VARIANT_REPOSITORY_NAME,
+    DEB_FIXTURE_MIXED_REPOSITORY_NAME,
     DEB_INSTALLER_FIXTURE_SUMMARY,
     DEB_INSTALLER_SOURCE_FIXTURE_SUMMARY,
     DEB_PACKAGE_NAME,
     DEB_PACKAGE_RELEASE_COMPONENT_NAME,
+    DEB_PACKAGE_INDEX_NAME,
     DEB_RELEASE_ARCHITECTURE_NAME,
     DEB_REPORT_CODE_SKIP_COMPLETE,
     DEB_REPORT_CODE_SKIP_PACKAGE,
@@ -515,3 +517,40 @@ def test_sync_architecture_variant_fields(
     )
     assert packages
     assert {pkg.architecture for pkg in packages} == {"amd64"}
+
+
+@pytest.mark.parallel
+def test_sync_hybrid_format_preserves_all_architecture_packages(
+    deb_init_and_sync,
+    deb_get_content_types,
+):
+    remote_args = {
+        "distributions": "muspelheim",
+        "components": "asgard",
+    }
+
+    repo, _, task = deb_init_and_sync(
+        url=DEB_FIXTURE_MIXED_REPOSITORY_NAME,
+        remote_args=remote_args,
+        return_task=True,
+    )
+
+    assert repo.latest_version_href.endswith("/1/")
+    assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE)
+
+    packages = deb_get_content_types(
+        "apt_package_api",
+        DEB_PACKAGE_NAME,
+        repo,
+        repo.latest_version_href,
+    )
+    assert packages
+    assert any(pkg.architecture == "all" for pkg in packages)
+
+    package_indices = deb_get_content_types(
+        "apt_package_indices_api",
+        DEB_PACKAGE_INDEX_NAME,
+        repo,
+        repo.latest_version_href,
+    )
+    assert any("binary-all/Packages" in pi.relative_path for pi in package_indices)

pulp_deb/tests/functional/constants.py

@@ -39,6 +39,7 @@ def _clean_dict(d):
 DEB_FIXTURE_INVALID_REPOSITORY_NAME = "/debian-invalid/"
 DEB_FIXTURE_FLAT_REPOSITORY_NAME = "/debian-flat/"
 DEB_FIXTURE_VARIANT_REPOSITORY_NAME = "/debian-variant/"
+DEB_FIXTURE_MIXED_REPOSITORY_NAME = "/debian-mixed/"
 DEB_FIXTURE_BASE = "/"
 DEB_FIXTURE_COMPLEX_REPOSITORY_NAME = "/debian-complex-dists"
 DEB_FIXTURE_MISSING_ARCHITECTURE_REPOSITORY_NAME = "/debian-missing-architecture/"