Add advanced copy mechanism without dependency solving in Pulp3

pulp_deb/app/serializers/repository_serializers.py

@@ -27,15 +27,23 @@ class CopySerializer(serializers.Serializer):
         help_text=_("A JSON document describing sources, destinations, and content to be copied")
     )
 
+    structured = serializers.BooleanField(
+        help_text=_(
+            "Also copy any distributions, components, and releases as needed for any packages "
+            "being copied. This will allow for structured publications of the target repository."
+        ),
+        default=True,
+    )
+
     dependency_solving = serializers.BooleanField(
-        help_text=_("Also copy dependencies of the content being copied."), default=False
+        help_text=_("Also copy dependencies of any packages being copied."), default=False
     )
 
     def validate(self, data):
         """
         Validate that the Serializer contains valid data.
-        Set the AptRepository based on the RepositoryVersion if only the latter is provided.
-        Set the RepositoryVersion based on the AptRepository if only the latter is provided.
+        Set the DebRepository based on the RepositoryVersion if only the latter is provided.
+        Set the RepositoryVersion based on the DebRepository if only the latter is provided.
         Convert the human-friendly names of the content types into what Pulp needs to query on.
         """
         super().validate(data)

pulp_deb/app/tasks/copy.py

@@ -3,17 +3,64 @@
 
 from pulpcore.plugin.models import Content, RepositoryVersion
 
+<<<<<<< HEAD
 from pulp_deb.app.depsolving import Solver
 from pulp_deb.app.models import AptRepository
+=======
+from pulp_deb.app.models import AptRepository, Package, ReleaseArchitecture, PackageReleaseComponent
+>>>>>>> 5ce018f... Add advanced copy mechanism without dependency solving in Pulp3
 
 import logging
 from gettext import gettext as _
 
 log = logging.getLogger(__name__)
 
 
+<<<<<<< HEAD
 @transaction.atomic
 def copy_content(config, dependency_solving=False):
+=======
+def find_structured_publish_attributes(content, src_repo_version):
+    """Finds the content for structured publish from packages to be copied and returns it all together.
+
+    Args:
+        content (iterable): Content for structured publish
+        src_repo_version (pulpcore.models.RepositoryVersion): Source repo version
+
+    Returns: Queryset of Content objects that extends intial set of content for structured publish
+    """
+
+    # Content in the source repository version
+    package_release_component_ids = src_repo_version.content.filter(
+        pulp_type=PackageReleaseComponent.get_pulp_type()
+    ).only("pk")
+    architecture_ids = src_repo_version.content.filter(
+        pulp_type=ReleaseArchitecture.get_pulp_type()
+    ).only("pk")
+    package_release_components = PackageReleaseComponent.objects.filter(
+        pk__in=package_release_component_ids
+    )
+
+    # Packages to be copied
+    packages = Package.objects.filter(pk__in=content)
+
+    children = set()
+
+    architectures = ReleaseArchitecture.objects.filter(pk__in=architecture_ids).values_list(
+        "pk", flat=True
+    )
+    children.update(architectures)
+
+    for pckg in package_release_components.iterator():
+        if pckg.package in packages:
+            children.update([pckg.pk, pckg.release_component.pk, pckg.release_component.release.pk])
+
+    return Content.objects.filter(pk__in=children)
+
+
+@transaction.atomic
+def copy_content(config, structured, dependency_solving):
+>>>>>>> 5ce018f... Add advanced copy mechanism without dependency solving in Pulp3
     """
     Copy content from one repo to another.
 
@@ -63,54 +110,14 @@ def process_entry(entry):
             ) = process_entry(entry)
 
             content_to_copy = source_repo_version.content.filter(content_filter)
+            if structured:
+                content_to_copy |= find_structured_publish_attributes(
+                    content_to_copy, source_repo_version
+                )
 
             base_version = dest_repo_version if dest_version_provided else None
+
             with dest_repo.new_version(base_version=base_version) as new_version:
                 new_version.add_content(content_to_copy)
     else:
-        # Dependency Solving Branch
-        # =========================
-
-        # TODO: a more structured way to store this state would be nice.
-        content_to_copy = {}
-        repo_mapping = {}
-        libsolv_repo_names = {}
-        base_versions = {}
-
-        solver = Solver()
-
-        for entry in config:
-            (
-                source_repo_version,
-                dest_repo_version,
-                dest_repo,
-                content_filter,
-                dest_version_provided,
-            ) = process_entry(entry)
-
-            repo_mapping[source_repo_version] = dest_repo_version
-            base_versions[source_repo_version] = dest_version_provided
-
-            # Load the content from the source and destination repository versions into the solver
-            source_repo_name = solver.load_source_repo(source_repo_version)
-            solver.load_target_repo(dest_repo_version)
-
-            # Store the correspondance between the libsolv name of a repo version and the
-            # actual Pulp repo version, so that we can work backwards to get the latter
-            # from the former.
-            libsolv_repo_names[source_repo_name] = source_repo_version
-
-            # Find all of the matching content in the repository version
-            content = source_repo_version.content.filter(content_filter)
-            content_to_copy[source_repo_name] = content
-
-        solver.finalize()
-
-        content_to_copy = solver.resolve_dependencies(content_to_copy)
-
-        for from_repo, units in content_to_copy.items():
-            src_repo_version = libsolv_repo_names[from_repo]
-            dest_repo_version = repo_mapping[src_repo_version]
-            base_version = dest_repo_version if base_versions[src_repo_version] else None
-            with dest_repo_version.repository.new_version(base_version=base_version) as new_version:
-                new_version.add_content(Content.objects.filter(pk__in=units))
+        raise NotImplementedError("Advanced copy with dependency solving is not yet implemented.")

pulp_deb/app/viewsets/repository.py

@@ -99,16 +99,17 @@ def create(self, request):
         serializer = serializers.CopySerializer(data=request.data, context={"request": request})
         serializer.is_valid(raise_exception=True)
 
-        dependency_solving = serializer.validated_data["dependency_solving"]
         config = serializer.validated_data["config"]
+        structured = serializer.validated_data["structured"]
+        dependency_solving = serializer.validated_data["dependency_solving"]
 
         config, shared_repos, exclusive_repos = self._process_config(config)
 
         async_result = dispatch(
             tasks.copy_content,
             shared_resources=shared_repos,
             exclusive_resources=exclusive_repos,
-            args=[config, dependency_solving],
+            args=[config, structured, dependency_solving],
             kwargs={},
         )
         return OperationPostponedResponse(async_result, request)