Merge pull request #1026 from ATIX-AG/fix-mirror-optimize

Add mirror option to optimize sync

CHANGES/1027.feature

@@ -0,0 +1 @@
+Allow optimize with mirror mode if nothing at all has changed in the repository being synced.

pulp_deb/app/tasks/synchronizing.py

@@ -185,6 +185,36 @@ def synchronize(remote_pk, repository_pk, mirror, optimize):
     if not remote.url:
         raise ValueError(_("A remote must have a url specified to synchronize."))
 
+    if optimize and mirror:
+        skip_dist = []
+        for dist in remote.distributions.split():
+            artifact_set_sha256 = get_distribution_release_file_artifact_set_sha256(dist, remote)
+            previous_release_file = get_previous_release_file(previous_repo_version, dist)
+            if (
+                previous_release_file
+                and previous_release_file.artifact_set_sha256 == artifact_set_sha256
+            ):
+                skip_dist.append(True)
+            else:
+                skip_dist.append(False)
+
+        remote_options = gen_remote_options(remote)
+        if not previous_repo_version.info:
+            optimize = False
+        elif not previous_repo_version.info["remote_options"] == remote_options:
+            optimize = False
+        elif not previous_repo_version.info["sync_options"]["mirror"] and mirror:
+            optimize = False
+
+        if all(skip_dist) and optimize:
+            log.info("No change in ReleaseFiles detected. Skipping sync.")
+            with ProgressReport(
+                message="Skipping sync (no changes for any ReleaseFile)",
+                code="sync.complete_skip.was_skipped",
+            ) as pb:
+                asyncio.run(pb.aincrement())
+            return
+
     first_stage = DebFirstStage(remote, optimize, mirror, previous_repo_version)
     DebDeclarativeVersion(first_stage, repository, mirror=mirror).create()
 
@@ -547,7 +577,7 @@ def __init__(self, remote, optimize, mirror, previous_repo_version, *args, **kwa
         self.optimize = optimize
         self.previous_repo_version = previous_repo_version
         self.sync_info = defaultdict()
-        self.sync_info["remote_options"] = self._gen_remote_options()
+        self.sync_info["remote_options"] = gen_remote_options(self.remote)
         self.sync_info["sync_options"] = {
             "optimize": optimize,
             "mirror": mirror,
@@ -599,19 +629,6 @@ def _to_d_artifact(self, relative_path, data=None):
             deferred_download=False,
         )
 
-    def _gen_remote_options(self):
-        return {
-            "distributions": self.remote.distributions,
-            "components": self.remote.components,
-            "architectures": self.remote.architectures,
-            "policy": self.remote.policy,
-            "sync_sources": self.remote.sync_sources,
-            "sync_udebs": self.remote.sync_udebs,
-            "sync_installer": self.remote.sync_installer,
-            "gpgkey": self.remote.gpgkey,
-            "ignore_missing_package_indices": self.remote.ignore_missing_package_indices,
-        }
-
     async def _handle_distribution(self, distribution):
         log.info(_('Downloading Release file for distribution: "{}"').format(distribution))
         # Create release_file
@@ -1254,8 +1271,7 @@ def _readd_previous_package_indices(previous_version, new_version, distribution)
     )
 
 
-@sync_to_async
-def _get_previous_release_file(previous_version, distribution):
+def get_previous_release_file(previous_version, distribution):
     previous_release_file_qs = previous_version.get_content(
         ReleaseFile.objects.filter(distribution=distribution)
     )
@@ -1265,6 +1281,9 @@ def _get_previous_release_file(previous_version, distribution):
     return previous_release_file_qs.first()
 
 
+_get_previous_release_file = sync_to_async(get_previous_release_file)
+
+
 @sync_to_async
 def _get_previous_package_index(previous_version, relative_path):
     previous_package_index_qs = previous_version.get_content(
@@ -1365,3 +1384,43 @@ def _get_source_checksums(source_paragraph, name):
         for checksum_type in settings.ALLOWED_CONTENT_CHECKSUMS
         if checksum_type in checksums
     }
+
+
+def gen_remote_options(remote):
+    return {
+        "distributions": remote.distributions,
+        "components": remote.components,
+        "architectures": remote.architectures,
+        "policy": remote.policy,
+        "sync_sources": remote.sync_sources,
+        "sync_udebs": remote.sync_udebs,
+        "sync_installer": remote.sync_installer,
+        "gpgkey": remote.gpgkey,
+        "ignore_missing_package_indices": remote.ignore_missing_package_indices,
+    }
+
+
+def get_distribution_release_file_artifact_set_sha256(distribution, remote):
+    log.info(_('Downloading Release file for distribution: "{}"').format(distribution))
+    if distribution[-1] == "/":
+        release_file_dir = distribution.strip("/")
+    else:
+        release_file_dir = os.path.join("dists", distribution)
+
+    release_file_info_serialized = {}
+    base_url = os.path.join(remote.url, release_file_dir)
+    for filename in ReleaseFile.SUPPORTED_ARTIFACTS:
+        url = os.path.join(base_url, filename)
+        downloader = remote.get_downloader(url=url)
+        try:
+            result = downloader.fetch()
+        except Exception:
+            continue
+        sha256 = result.artifact_attributes["sha256"]
+        release_file_info_serialized[filename] = sha256
+
+    hash_string = ""
+    for filename, sha256 in release_file_info_serialized.items():
+        hash_string = hash_string + filename + "," + sha256 + "\n"
+
+    return hashlib.sha256(hash_string.encode("utf-8")).hexdigest()

pulp_deb/tests/functional/api/test_sync.py

@@ -16,6 +16,7 @@
     DEB_INSTALLER_SOURCE_FIXTURE_SUMMARY,
     DEB_REPORT_CODE_SKIP_PACKAGE,
     DEB_REPORT_CODE_SKIP_RELEASE,
+    DEB_REPORT_CODE_SKIP_COMPLETE,
     DEB_SIGNING_KEY,
 )
 from pulp_deb.tests.functional.utils import get_counts_from_content_summary
@@ -311,9 +312,7 @@ def test_sync_optimize_skip_unchanged_package_index(
 
 
 @pytest.mark.parallel
-def test_sync_optimize_switch_to_no_mirror(
-    deb_init_and_sync,
-):
+def test_sync_optimize_switch_to_no_mirror(deb_init_and_sync):
     """
     Test that when syncing a repo with mirror=True, and then re-syncing that repo with
     mirror=False, optimize=True, the releases will be skipped by optimize mode.
@@ -331,6 +330,23 @@ def test_sync_optimize_switch_to_no_mirror(
     assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE)
 
 
+@pytest.mark.parallel
+def test_sync_optimize_with_mirror_enabled(deb_init_and_sync):
+    """Test if enabling mirror sync option will skip syncing (optimize) on resync."""
+
+    sync_args = {"mirror": True}
+    repo, remote, task = deb_init_and_sync(sync_args=sync_args, return_task=True)
+    assert repo.latest_version_href.endswith("/1/")
+    assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE)
+
+    # resync
+    repo, _, task = deb_init_and_sync(
+        repository=repo, remote=remote, sync_args=sync_args, return_task=True
+    )
+    assert repo.latest_version_href.endswith("/1/")
+    assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE)
+
+
 def test_sync_orphan_cleanup_fail(
     deb_init_and_sync,
     orphans_cleanup_api_client,

pulp_deb/tests/functional/constants.py

@@ -213,6 +213,7 @@ def _clean_dict(d):
 
 DEB_FIXTURE_PACKAGE_COUNT = DEB_FIXTURE_SUMMARY.get(DEB_PACKAGE_NAME, 0)
 
+DEB_REPORT_CODE_SKIP_COMPLETE = "sync.complete_skip.was_skipped"
 DEB_REPORT_CODE_SKIP_RELEASE = "sync.release_file.was_skipped"
 DEB_REPORT_CODE_SKIP_PACKAGE = "sync.package_index.was_skipped"