Add depency resolving on advanced copy endpoint

CHANGES/386.feature

@@ -0,0 +1,6 @@
+Implemented ``dependency_solving`` on the advanced copy endpoint. When enabled, the
+copy task BFS-walks the Depends/Pre-Depends closure of every initial package, picks
+the newest version satisfying each relation from the source repository, honours
+Provides aliases, and skips relations already satisfied by the destination base
+version. Unsatisfiable relations raise an explicit error. Resolves
+:redmine:`386`.

pulp_deb/app/serializers/repository_serializers.py

@@ -271,8 +271,9 @@ class CopySerializer(ValidateFieldsMixin, serializers.Serializer):
 
     dependency_solving = serializers.BooleanField(
         help_text=_(
-            "Also copy dependencies of any packages being copied. NOT YET"
-            'IMPLEMENTED! You must keep this at "False"!'
+            "Also copy the transitive Depends/Pre-Depends closure of every package being copied. "
+            "Already-satisfied relations (present in dest_base_version) are skipped. "
+            "Raises an error if any relation cannot be satisfied from the source repository."
         ),
         default=False,
     )

pulp_deb/app/tasks/copy.py

@@ -4,7 +4,6 @@
 from django.db import transaction
 from django.db.models import Q
 
-from pulpcore.plugin.exceptions import FeatureNotImplementedError
 from pulpcore.plugin.models import RepositoryVersion
 from pulpcore.plugin.util import get_domain_pk
 
@@ -15,6 +14,7 @@
     Release,
     ReleaseArchitecture,
 )
+from pulp_deb.app.tasks.dependency_solving import solve_dependencies
 
 log = logging.getLogger(__name__)
 
@@ -105,11 +105,6 @@ def process_entry(entry):
             content_filter,
         )
 
-    if dependency_solving:
-        raise FeatureNotImplementedError(
-            "Advanced copy with dependency solving is not yet implemented."
-        )
-
     for entry in config:
         (
             source_repo_version,
@@ -119,6 +114,16 @@ def process_entry(entry):
         ) = process_entry(entry)
 
         content_to_copy = source_repo_version.content.filter(content_filter)
+
+        if dependency_solving:
+            initial_packages = Package.objects.filter(
+                pk__in=content_to_copy.filter(pulp_type=Package.get_pulp_type()).only("pk")
+            )
+            extra_pks = solve_dependencies(initial_packages, source_repo_version, dest_base_version)
+            content_to_copy = source_repo_version.content.filter(
+                Q(pk__in=content_to_copy.values("pk")) | Q(pk__in=extra_pks)
+            )
+
         if structured:
             content_to_copy = find_structured_publish_content(content_to_copy, source_repo_version)
 

pulp_deb/app/tasks/dependency_solving.py

@@ -0,0 +1,160 @@
+from collections import deque
+from gettext import gettext as _
+
+from debian.deb822 import PkgRelation
+from debian.debian_support import Version
+from django.db.models import Q
+
+from pulp_deb.app.models import Package
+
+
+class DependencySolveError(Exception):
+    """Raised when a transitive dependency cannot be satisfied from the source repository."""
+
+
+_OP_FUNCS = {
+    ">=": lambda a, b: a >= b,
+    ">>": lambda a, b: a > b,
+    "<=": lambda a, b: a <= b,
+    "<<": lambda a, b: a < b,
+    "=": lambda a, b: a == b,
+}
+
+
+def _parse_relations(*depends_strings):
+    combined = ", ".join(s for s in depends_strings if s)
+    if not combined.strip():
+        return []
+    try:
+        return PkgRelation.parse_relations(combined)
+    except Exception:
+        return []
+
+
+def _version_satisfies(pkg_version, constraint):
+    if not constraint:
+        return True
+    op, target = constraint
+    return _OP_FUNCS.get(op, lambda a, b: False)(Version(pkg_version), Version(target))
+
+
+def _candidates_for(alt, package_qs, pkg_arch=None):
+    name = alt["name"]
+    qs = package_qs.filter(package=name)
+    alt_arch = alt.get("arch")
+    if alt_arch:
+        qs = qs.filter(architecture=alt_arch)
+    elif pkg_arch:
+        # Restrict deps to the consumer's arch or arch-independent (`all`).
+        qs = qs.filter(Q(architecture=pkg_arch) | Q(architecture="all"))
+    return qs
+
+
+def _find_satisfier(alternative, package_qs, pkg_arch=None):
+    """Best Package satisfying a single OR-alternative (direct hit or via Provides)."""
+    constraint = alternative.get("version")
+
+    direct = list(_candidates_for(alternative, package_qs, pkg_arch))
+    direct.sort(key=lambda p: Version(p.version), reverse=True)
+    for pkg in direct:
+        if _version_satisfies(pkg.version, constraint):
+            return pkg
+
+    # Provides: a Package P providing the name we need.  Versioned Provides
+    # are rare; we honour them when present, otherwise treat as unversioned.
+    name = alternative["name"]
+    provides_qs = package_qs.exclude(provides__isnull=True).exclude(provides="")
+    if pkg_arch:
+        provides_qs = provides_qs.filter(Q(architecture=pkg_arch) | Q(architecture="all"))
+    for pkg in provides_qs:
+        for or_group in _parse_relations(pkg.provides):
+            for prov in or_group:
+                if prov["name"] != name:
+                    continue
+                prov_version = prov.get("version")
+                if not constraint:
+                    return pkg
+                if (
+                    prov_version
+                    and prov_version[0] == "="
+                    and _version_satisfies(prov_version[1], constraint)
+                ):
+                    return pkg
+                # Unversioned Provides cannot satisfy a versioned dependency per policy.
+    return None
+
+
+def _relation_satisfied_by_set(relation, package_qs, pkg_arch=None):
+    return any(_find_satisfier(alt, package_qs, pkg_arch) is not None for alt in relation)
+
+
+def _packages_in(repo_version):
+    if repo_version is None:
+        return Package.objects.none()
+    return Package.objects.filter(
+        pk__in=repo_version.content.filter(pulp_type=Package.get_pulp_type()).only("pk")
+    )
+
+
+def _format_relation(relation):
+    parts = []
+    for alt in relation:
+        s = alt["name"]
+        if alt.get("arch"):
+            s += ":{}".format(alt["arch"])
+        if alt.get("version"):
+            s += " ({} {})".format(*alt["version"])
+        parts.append(s)
+    return " | ".join(parts)
+
+
+def solve_dependencies(initial_packages, source_repo_version, dest_base_version=None):
+    """BFS over Depends + Pre-Depends. Returns the set of Package pks to copy.
+
+    A relation already satisfied by ``dest_base_version`` is skipped; otherwise
+    the newest satisfying Package in ``source_repo_version`` is chosen, then
+    walked transitively.  Raises :class:`DependencySolveError` on unsatisfiable
+    relations.
+    """
+    source_pkgs = _packages_in(source_repo_version)
+    dest_pkgs = _packages_in(dest_base_version)
+
+    seen = set()
+    queue = deque(initial_packages)
+    final = set()
+
+    while queue:
+        pkg = queue.popleft()
+        if pkg.pk in seen:
+            continue
+        seen.add(pkg.pk)
+        final.add(pkg.pk)
+
+        for relation in _parse_relations(pkg.depends, pkg.pre_depends):
+            if dest_base_version is not None and _relation_satisfied_by_set(
+                relation, dest_pkgs, pkg.architecture
+            ):
+                continue
+
+            chosen = None
+            for alt in relation:
+                chosen = _find_satisfier(alt, source_pkgs, pkg.architecture)
+                if chosen:
+                    break
+
+            if chosen is None:
+                raise DependencySolveError(
+                    _(
+                        "Cannot solve dependency for {pkg}: "
+                        "'{rel}' is not satisfiable in source repository version {srv}"
+                    ).format(
+                        pkg=pkg.name,
+                        rel=_format_relation(relation),
+                        srv=source_repo_version.pk,
+                    )
+                )
+
+            if chosen.pk not in seen:
+                queue.append(chosen)
+
+    return final

pulp_deb/tests/functional/api/test_copy.py

@@ -76,3 +76,32 @@ def test_copy_empty_content(
 
     target_repo = deb_get_repository_by_href(target_repo.pulp_href)
     assert target_repo.latest_version_href.endswith("/versions/0/")
+
+
+@pytest.mark.parallel
+def test_copy_with_dependency_solving(
+    deb_init_and_sync,
+    deb_repository_factory,
+    apt_package_api,
+    deb_copy_content,
+    deb_get_repository_by_href,
+    deb_get_content_summary,
+):
+    """Smoke test that ``dependency_solving=True`` reaches the new code path."""
+    source_repo, _ = deb_init_and_sync()
+    target_repo = deb_repository_factory()
+    package = apt_package_api.list(
+        package="frigg",
+        repository_version=source_repo.latest_version_href,
+    ).results[0]
+
+    deb_copy_content(
+        source_repo_version=source_repo.latest_version_href,
+        dest_repo=target_repo.pulp_href,
+        content=[package.pulp_href],
+        dependency_solving=True,
+    )
+
+    target_repo = deb_get_repository_by_href(target_repo.pulp_href)
+    added_summary = deb_get_content_summary(target_repo).added
+    assert DEB_ADVANCED_COPY_FIXTURE_SUMMARY == get_counts_from_content_summary(added_summary)

pulp_deb/tests/functional/conftest.py

@@ -491,19 +491,26 @@ def _deb_acs_factory(**kwargs):
 def deb_copy_content(apt_copy_api, monitor_task):
     """Fixture that copies deb content from a source repository version to a target repository."""
 
-    def _deb_copy_content(source_repo_version, dest_repo, content=None, structured=True):
+    def _deb_copy_content(
+        source_repo_version,
+        dest_repo,
+        content=None,
+        structured=True,
+        dependency_solving=False,
+    ):
         """Copy deb content from a source repository version to a target repository.
 
         :param source_repo_version: The repository version href from where the content is copied.
         :dest_repo: The repository href where the content should be copied to.
         :content: List of package hrefs that should be copied from the source. Default: None
         :structured: Whether or not the content should be structured copied. Default: True
+        :dependency_solving: Also copy the Depends/Pre-Depends closure. Default: False
         :returns: The task of the copy operation.
         """
         config = {"source_repo_version": source_repo_version, "dest_repo": dest_repo}
         if content is not None:
             config["content"] = content
-        data = Copy(config=[config], structured=structured)
+        data = Copy(config=[config], structured=structured, dependency_solving=dependency_solving)
         response = apt_copy_api.copy_content(data)
         return monitor_task(response.task)