Merge pull request release-engineering#311 from rajulkumar/del_mod_rp…

…m_all_repo

Remove rpms in the modules from all the provided repos [RHELDST-13651]

src/pubtools/_pulp/tasks/delete.py

@@ -234,14 +234,19 @@ def delete_modules(self, repos, module_names, signing_keys=None):
         f = f_map(mods_f, partial(self.map_to_repo, repos=repos, unit_attr="nsvca"))
         repo_map_f = f_map(f, self.log_units)
 
-        # remove packages from modules
+        # remove packages in the modules from all the provided repos,
+        # after establishing that the modules are present in one of the
+        # provided repos in the previous step
         unit_map_f = f_map(f, lambda map: map[1])
         artifact_f = f_map(
-            unit_map_f, partial(self.remove_mod_artifacts, signing_keys=signing_keys)
+            unit_map_f,
+            lambda unit_map: self.remove_mod_artifacts(
+                unit_map.values(), repos=repos, signing_keys=signing_keys
+            ),
         )
 
         # hold for rpms in the artifacts to be removed before removing modules
-        # wait for rpms_fs to resolve for f_zip to resolve and return repo_map_f
+        # wait for artifact_f to resolve for f_zip to resolve and return repo_map_f
         # to remove_modules in next step
         repo_map_f = f_map(
             f_zip(repo_map_f, f_sequence(artifact_f.result())), lambda t: t[0]
@@ -389,11 +394,10 @@ def verify_repos_from_advisory(self, advisory, repos):
         return verified_repos
 
     @step("Remove artifacts from modules")
-    def remove_mod_artifacts(self, unit_map, signing_keys):
+    def remove_mod_artifacts(self, modules, repos, signing_keys):
         out = []
-        for item in unit_map.values():
+        for item in modules:
             rpms = item.unit.artifacts_filenames
-            repos = item.repos
             if rpms:
                 rpms_info = [RpmInfoItem(filename=rpm, sha256sum=None) for rpm in rpms]
                 out.append(self.delete_rpms(repos, rpms_info, signing_keys))

tests/delete/test_delete_packages.py

@@ -390,6 +390,11 @@ def test_delete_modules(command_tester, fake_collector, monkeypatch):
     repo = YumRepository(
         id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
     )
+    repo2 = YumRepository(
+        id="other-yumrepo",
+        relative_url="other/publish/url",
+        mutable_urls=["repomd.xml"],
+    )
 
     files = [
         RpmUnit(
@@ -424,14 +429,49 @@ def test_delete_modules(command_tester, fake_collector, monkeypatch):
             version=123,
             context="a1c2",
             arch="s390x",
-            artifacts=["bash-0:1.23-1.test8_x86_64", "dash-0:1.23-1.test8_x86_64"],
+            artifacts=[
+                "bash-0:1.23-1.test8_x86_64",
+                "dash-0:1.23-1.test8_x86_64",
+                "smash-0.24-1.test8_x86_64",
+            ],
             unit_id="module1",
         ),
     ]
 
+    files2 = [
+        RpmUnit(
+            name="smash",
+            version="0.24",
+            release="1.test8",
+            arch="x86_64",
+            filename="smash-0.24-1.test8_x86_64.rpm",
+            sha256sum="a" * 64,
+            md5sum="b" * 32,
+            signing_key="aabbcc",
+            provides=[],
+            requires=[],
+            unit_id="rpm3",
+        ),
+        RpmUnit(
+            name="rash",
+            version="2.23",
+            release="1.test8",
+            arch="x86_64",
+            filename="rash-2.23-1.test8_x86_64.rpm",
+            sha256sum="a" * 64,
+            md5sum="b" * 32,
+            signing_key="aabbcc",
+            provides=[],
+            requires=[],
+            unit_id="rpm4",
+        ),
+    ]
+
     with FakeDeletePackages() as task_instance:
         task_instance.pulp_client_controller.insert_repository(repo)
+        task_instance.pulp_client_controller.insert_repository(repo2)
         task_instance.pulp_client_controller.insert_units(repo, files)
+        task_instance.pulp_client_controller.insert_units(repo2, files2)
 
         # It should run with expected output.
         command_tester.test(
@@ -442,6 +482,8 @@ def test_delete_modules(command_tester, fake_collector, monkeypatch):
                 "https://pulp.example.com/",
                 "--repo",
                 "some-yumrepo",
+                "--repo",
+                "other-yumrepo",
                 "--file",
                 "mymod:s1:123:a1c2:s390x",
                 "--signing-key",
@@ -480,32 +522,62 @@ def test_delete_modules(command_tester, fake_collector, monkeypatch):
                 "signing_key": None,
                 "filename": "mymod:s1:123:a1c2:s390x",
             },
+            {
+                "origin": "pulp",
+                "src": None,
+                "state": "DELETED",
+                "build": None,
+                "dest": "other-yumrepo",
+                "checksums": {"sha256": "a" * 64},
+                "signing_key": None,
+                "filename": "smash-0.24-1.test8.x86_64.rpm",
+            },
         ]
 
         # verify whether files were deleted on Pulp
         client = task_instance.pulp_client
 
-        # get the repo where the files were deleted
+        # get the repos where the files were deleted
         repos = list(
             client.search_repository(Criteria.with_id("some-yumrepo")).result()
         )
         assert len(repos) == 1
         repo = repos[0]
 
+        repos2 = list(
+            client.search_repository(Criteria.with_id("other-yumrepo")).result()
+        )
+        assert len(repos2) == 1
+        repo2 = repos2[0]
+
         # criteria with the unit_ids
         unit_ids = []
         for f in files:
             unit_ids.append(f.unit_id)
         criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
 
-        # deleted files are not in the repo
+        unit_ids2 = []
+        for f in files2:
+            unit_ids2.append(f.unit_id)
+        criteria2 = Criteria.with_field("unit_id", Matcher.in_(unit_ids2))
+
+        # deleted files are not in "some-yumrepo" repo
         files = list(repo.search_content(criteria).result())
         assert len(files) == 0
 
+        # there's one file in "other-yumrepo" repo as only one was
+        # listed in module's artifacts and was deleted with the module
+        files2 = list(repo2.search_content(criteria2).result())
+        assert len(files2) == 1
+        assert files2[0].filename == "rash-2.23-1.test8_x86_64.rpm"
+
         # same files exist on Pulp as orphans
         files_search = list(client.search_content(criteria).result())
         assert len(files_search) == 3
 
+        files_search = list(client.search_content(criteria2).result())
+        assert len(files_search) == 2
+
 
 def test_delete_files(command_tester, fake_collector, monkeypatch):
     """Deleting files from repos succeeds"""

tests/logs/delete/test_delete_packages/test_delete_modules.txt

@@ -3,7 +3,7 @@
 [ WARNING] Requested unit(s) don't exist as file: mymod:s1:123:a1c2:s390x
 [    INFO] 0 unit(s) found for deletion
 [    INFO] Get files: finished
-[ WARNING] No units to remove from some-yumrepo
+[ WARNING] No units to remove from other-yumrepo, some-yumrepo
 [    INFO] Unassociate files: started
 [ WARNING] Nothing mapped for removal
 [    INFO] Unassociate files: finished
@@ -14,23 +14,30 @@
 [    INFO] Get modules: started
 [    INFO] 1 unit(s) found for deletion
 [    INFO] Get modules: finished
+[ WARNING] mymod:s1:123:a1c2:s390x is not present in other-yumrepo
+[ WARNING] No units to remove from other-yumrepo
 [    INFO] Deleting mymod:s1:123:a1c2:s390x from some-yumrepo
 [    INFO] Remove artifacts from modules: started
 [    INFO] Delete RPMs: started
 [    INFO] Get RPMs: started
-[    INFO] 2 unit(s) found for deletion
+[    INFO] 3 unit(s) found for deletion
 [    INFO] Get RPMs: finished
+[ WARNING] bash-1.23-1.test8_x86_64.rpm is not present in other-yumrepo
+[ WARNING] dash-1.23-1.test8_x86_64.rpm is not present in other-yumrepo
+[ WARNING] smash-0.24-1.test8_x86_64.rpm is not present in some-yumrepo
 [    INFO] Deleting bash-1.23-1.test8_x86_64.rpm from some-yumrepo
 [    INFO] Deleting dash-1.23-1.test8_x86_64.rpm from some-yumrepo
+[    INFO] Deleting smash-0.24-1.test8_x86_64.rpm from other-yumrepo
 [    INFO] Unassociate RPMs: started
-[    INFO] some-yumrepo: removed 2 rpm(s), tasks: e3e70682-c209-4cac-629f-6fbed82c07cd
+[    INFO] other-yumrepo: removed 1 rpm(s), tasks: e3e70682-c209-4cac-629f-6fbed82c07cd
+[    INFO] some-yumrepo: removed 2 rpm(s), tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34
 [    INFO] Unassociate RPMs: finished
 [    INFO] Record push items: started
 [    INFO] Record push items: finished
 [    INFO] Delete RPMs: finished
 [    INFO] Remove artifacts from modules: finished
 [    INFO] Unassociate modules: started
-[    INFO] some-yumrepo: removed 1 modulemd(s), tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34
+[    INFO] some-yumrepo: removed 1 modulemd(s), tasks: d4713d60-c8a7-0639-eb11-67b367a9c378
 [    INFO] Unassociate modules: finished
 [    INFO] Record push items: started
 [    INFO] Record push items: finished