From 7fc41b8aa41d1e931284c960738b8f50c15984af Mon Sep 17 00:00:00 2001 From: Tobias Grigo Date: Wed, 31 Jan 2024 17:04:00 +0100 Subject: [PATCH] Add mirror option to optimize sync closes #1027 --- CHANGES/1027.feature | 1 + pulp_deb/app/tasks/synchronizing.py | 91 ++++++++++++++++++---- pulp_deb/tests/functional/api/test_sync.py | 22 +++++- pulp_deb/tests/functional/constants.py | 1 + 4 files changed, 96 insertions(+), 19 deletions(-) create mode 100644 CHANGES/1027.feature diff --git a/CHANGES/1027.feature b/CHANGES/1027.feature new file mode 100644 index 000000000..448fa638a --- /dev/null +++ b/CHANGES/1027.feature @@ -0,0 +1 @@ +Allow optimize with mirror mode if nothing at all has changed in the repository being synced. diff --git a/pulp_deb/app/tasks/synchronizing.py b/pulp_deb/app/tasks/synchronizing.py index 869814b30..2b5b99479 100644 --- a/pulp_deb/app/tasks/synchronizing.py +++ b/pulp_deb/app/tasks/synchronizing.py @@ -185,6 +185,36 @@ def synchronize(remote_pk, repository_pk, mirror, optimize): if not remote.url: raise ValueError(_("A remote must have a url specified to synchronize.")) + if optimize and mirror: + skip_dist = [] + for dist in remote.distributions.split(): + artifact_set_sha256 = get_distribution_release_file_artifact_set_sha256(dist, remote) + previous_release_file = get_previous_release_file(previous_repo_version, dist) + if ( + previous_release_file + and previous_release_file.artifact_set_sha256 == artifact_set_sha256 + ): + skip_dist.append(True) + else: + skip_dist.append(False) + + remote_options = gen_remote_options(remote) + if not previous_repo_version.info: + optimize = False + elif not previous_repo_version.info["remote_options"] == remote_options: + optimize = False + elif not previous_repo_version.info["sync_options"]["mirror"] and mirror: + optimize = False + + if all(skip_dist) and optimize: + log.info("No change in ReleaseFiles detected. Skipping sync.") + with ProgressReport( + message="Skipping sync (no changes for any ReleaseFile)", + code="sync.complete_skip.was_skipped", + ) as pb: + asyncio.run(pb.aincrement()) + return + first_stage = DebFirstStage(remote, optimize, mirror, previous_repo_version) DebDeclarativeVersion(first_stage, repository, mirror=mirror).create() @@ -547,7 +577,7 @@ def __init__(self, remote, optimize, mirror, previous_repo_version, *args, **kwa self.optimize = optimize self.previous_repo_version = previous_repo_version self.sync_info = defaultdict() - self.sync_info["remote_options"] = self._gen_remote_options() + self.sync_info["remote_options"] = gen_remote_options(self.remote) self.sync_info["sync_options"] = { "optimize": optimize, "mirror": mirror, @@ -599,19 +629,6 @@ def _to_d_artifact(self, relative_path, data=None): deferred_download=False, ) - def _gen_remote_options(self): - return { - "distributions": self.remote.distributions, - "components": self.remote.components, - "architectures": self.remote.architectures, - "policy": self.remote.policy, - "sync_sources": self.remote.sync_sources, - "sync_udebs": self.remote.sync_udebs, - "sync_installer": self.remote.sync_installer, - "gpgkey": self.remote.gpgkey, - "ignore_missing_package_indices": self.remote.ignore_missing_package_indices, - } - async def _handle_distribution(self, distribution): log.info(_('Downloading Release file for distribution: "{}"').format(distribution)) # Create release_file @@ -1254,8 +1271,7 @@ def _readd_previous_package_indices(previous_version, new_version, distribution) ) -@sync_to_async -def _get_previous_release_file(previous_version, distribution): +def get_previous_release_file(previous_version, distribution): previous_release_file_qs = previous_version.get_content( ReleaseFile.objects.filter(distribution=distribution) ) @@ -1265,6 +1281,9 @@ def _get_previous_release_file(previous_version, distribution): return previous_release_file_qs.first() +_get_previous_release_file = sync_to_async(get_previous_release_file) + + @sync_to_async def _get_previous_package_index(previous_version, relative_path): previous_package_index_qs = previous_version.get_content( @@ -1365,3 +1384,43 @@ def _get_source_checksums(source_paragraph, name): for checksum_type in settings.ALLOWED_CONTENT_CHECKSUMS if checksum_type in checksums } + + +def gen_remote_options(remote): + return { + "distributions": remote.distributions, + "components": remote.components, + "architectures": remote.architectures, + "policy": remote.policy, + "sync_sources": remote.sync_sources, + "sync_udebs": remote.sync_udebs, + "sync_installer": remote.sync_installer, + "gpgkey": remote.gpgkey, + "ignore_missing_package_indices": remote.ignore_missing_package_indices, + } + + +def get_distribution_release_file_artifact_set_sha256(distribution, remote): + log.info(_('Downloading Release file for distribution: "{}"').format(distribution)) + if distribution[-1] == "/": + release_file_dir = distribution.strip("/") + else: + release_file_dir = os.path.join("dists", distribution) + + release_file_info_serialized = {} + base_url = os.path.join(remote.url, release_file_dir) + for filename in ReleaseFile.SUPPORTED_ARTIFACTS: + url = os.path.join(base_url, filename) + downloader = remote.get_downloader(url=url) + try: + result = downloader.fetch() + except Exception: + continue + sha256 = result.artifact_attributes["sha256"] + release_file_info_serialized[filename] = sha256 + + hash_string = "" + for filename, sha256 in release_file_info_serialized.items(): + hash_string = hash_string + filename + "," + sha256 + "\n" + + return hashlib.sha256(hash_string.encode("utf-8")).hexdigest() diff --git a/pulp_deb/tests/functional/api/test_sync.py b/pulp_deb/tests/functional/api/test_sync.py index ec0d07445..54ebb7c7a 100644 --- a/pulp_deb/tests/functional/api/test_sync.py +++ b/pulp_deb/tests/functional/api/test_sync.py @@ -16,6 +16,7 @@ DEB_INSTALLER_SOURCE_FIXTURE_SUMMARY, DEB_REPORT_CODE_SKIP_PACKAGE, DEB_REPORT_CODE_SKIP_RELEASE, + DEB_REPORT_CODE_SKIP_COMPLETE, DEB_SIGNING_KEY, ) from pulp_deb.tests.functional.utils import get_counts_from_content_summary @@ -311,9 +312,7 @@ def test_sync_optimize_skip_unchanged_package_index( @pytest.mark.parallel -def test_sync_optimize_switch_to_no_mirror( - deb_init_and_sync, -): +def test_sync_optimize_switch_to_no_mirror(deb_init_and_sync): """ Test that when syncing a repo with mirror=True, and then re-syncing that repo with mirror=False, optimize=True, the releases will be skipped by optimize mode. @@ -331,6 +330,23 @@ def test_sync_optimize_switch_to_no_mirror( assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE) +@pytest.mark.parallel +def test_sync_optimize_with_mirror_enabled(deb_init_and_sync): + """Test if enabling mirror sync option will skip syncing (optimize) on resync.""" + + sync_args = {"mirror": True} + repo, remote, task = deb_init_and_sync(sync_args=sync_args, return_task=True) + assert repo.latest_version_href.endswith("/1/") + assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE) + + # resync + repo, _, task = deb_init_and_sync( + repository=repo, remote=remote, sync_args=sync_args, return_task=True + ) + assert repo.latest_version_href.endswith("/1/") + assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE) + + def test_sync_orphan_cleanup_fail( deb_init_and_sync, orphans_cleanup_api_client, diff --git a/pulp_deb/tests/functional/constants.py b/pulp_deb/tests/functional/constants.py index 7f7d8efb8..8740cf366 100644 --- a/pulp_deb/tests/functional/constants.py +++ b/pulp_deb/tests/functional/constants.py @@ -213,6 +213,7 @@ def _clean_dict(d): DEB_FIXTURE_PACKAGE_COUNT = DEB_FIXTURE_SUMMARY.get(DEB_PACKAGE_NAME, 0) +DEB_REPORT_CODE_SKIP_COMPLETE = "sync.complete_skip.was_skipped" DEB_REPORT_CODE_SKIP_RELEASE = "sync.release_file.was_skipped" DEB_REPORT_CODE_SKIP_PACKAGE = "sync.package_index.was_skipped"