[Backport master] Add a tool to update the dpkg-versions

c2cciutils/__init__.py

@@ -572,3 +572,59 @@ def snyk_exec() -> tuple[str, dict[str, str]]:
         subprocess.run(["snyk", "config", "set", f"org={env['SNYK_ORG']}"], check=True, env=env)
 
     return os.path.join(os.path.dirname(os.path.abspath(__file__)), "node_modules/snyk/bin/snyk"), env
+
+
+def create_pull_request_if_needed(
+    current_branch: str,
+    new_branch: str,
+    commit_message: str,
+    pull_request_extra_arguments: Optional[list[str]] = None,
+) -> bool:
+    """
+    Create a pull request if there are changes.
+    """
+
+    if pull_request_extra_arguments is None:
+        pull_request_extra_arguments = []
+
+    diff_proc = subprocess.run(["git", "diff", "--quiet"])  # pylint: disable=subprocess-run-check
+    if diff_proc.returncode != 0:
+        print("::group::Diff")
+        sys.stdout.flush()
+        sys.stderr.flush()
+        subprocess.run(["git", "diff"], check=True)
+        print("::endgroup::")
+
+        git_hash = subprocess.run(
+            ["git", "rev-parse", "HEAD"], check=True, stdout=subprocess.PIPE, encoding="utf-8"
+        ).stdout.strip()
+        subprocess.run(["git", "checkout", "-b", new_branch], check=True)
+        subprocess.run(["git", "add", "--all"], check=True)
+        subprocess.run(["git", "commit", f"--message={commit_message}"], check=True)
+        if os.environ.get("TEST") != "TRUE":
+            subprocess.run(
+                ["git", "push", "--force", "origin", f"snyk-fix/{current_branch}"],
+                check=True,
+            )
+            env = os.environ.copy()
+            if "GH_TOKEN" not in env:
+                if "GITHUB_TOKEN" in env:
+                    env["GH_TOKEN"] = env["GITHUB_TOKEN"]
+                else:
+                    env["GH_TOKEN"] = str(c2cciutils.gopass("gs/ci/github/token/gopass"))
+            subprocess.run(
+                [
+                    "gh",
+                    "pr",
+                    "create",
+                    f"--base={current_branch}",
+                    *pull_request_extra_arguments,
+                ],
+                check=True,
+                env=env,
+            )
+        else:
+            subprocess.run(["git", "reset", "--hard"], check=True)
+        subprocess.run(["git", "checkout", git_hash], check=True)
+
+    return diff_proc.returncode != 0

c2cciutils/audit.py

@@ -162,54 +162,19 @@ def snyk(
     print("::endgroup::")
 
     if not args.fix:
-        diff_proc = subprocess.run(["git", "diff", "--quiet"])  # pylint: disable=subprocess-run-check
-        if diff_proc.returncode != 0:
-            print("::error::There is some changes to commit")
-            print("::group::Diff")
-            sys.stdout.flush()
-            sys.stderr.flush()
-            subprocess.run(["git", "diff"], check=True)
-            print("::endgroup::")
-
-            current_branch = c2cciutils.get_branch(args.branch)
-            git_hash = subprocess.run(
-                ["git", "rev-parse", "HEAD"], check=True, stdout=subprocess.PIPE, encoding="utf-8"
-            ).stdout.strip()
-            subprocess.run(["git", "checkout", "-b", f"snyk-fix/{current_branch}"], check=True)
-            subprocess.run(["git", "add", "--all"], check=True)
-            subprocess.run(["git", "commit", "--message=Snyk auto fix"], check=True)
-            if os.environ.get("TEST") != "TRUE":
-                subprocess.run(
-                    ["git", "push", "--force", "origin", f"snyk-fix/{current_branch}"],
-                    check=True,
-                )
-                env = os.environ.copy()
-                if "GH_TOKEN" not in env:
-                    if "GITHUB_TOKEN" in env:
-                        env["GH_TOKEN"] = env["GITHUB_TOKEN"]
-                    else:
-                        env["GH_TOKEN"] = str(c2cciutils.gopass("gs/ci/github/token/gopass"))
-                fix_github_create_pull_request_arguments = config.get(
-                    "fix_github_create_pull_request_arguments",
-                    c2cciutils.configuration.AUDIT_SNYK_FIX_PULL_REQUEST_ARGUMENTS_DEFAULT,
-                )
-                subprocess.run(
-                    [
-                        "gh",
-                        "pr",
-                        "create",
-                        f"--base={current_branch}",
-                        f"--body={snyk_fix_message}",
-                        *fix_github_create_pull_request_arguments,
-                    ],
-                    check=True,
-                    env=env,
-                )
-            else:
-                subprocess.run(["git", "reset", "--hard"], check=True)
-            subprocess.run(["git", "checkout", git_hash], check=True)
+        current_branch = c2cciutils.get_branch(args.branch)
+        fix_github_create_pull_request_arguments = config.get(
+            "fix_github_create_pull_request_arguments",
+            c2cciutils.configuration.AUDIT_SNYK_FIX_PULL_REQUEST_ARGUMENTS_DEFAULT,
+        )
+        has_diff = c2cciutils.create_pull_request_if_needed(
+            current_branch,
+            f"snyk-fix/{current_branch}",
+            "Snyk auto fix",
+            [f"--body={snyk_fix_message}", *fix_github_create_pull_request_arguments],
+        )
 
-    return install_success and test_success and diff_proc.returncode == 0
+    return install_success and test_success and not has_diff
 
 
 def outdated_versions(

c2cciutils/scripts/docker_versions_update.py

@@ -0,0 +1,97 @@
+import argparse
+import re
+import subprocess  # nosec
+import tempfile
+
+import ruamel.yaml
+
+import c2cciutils
+
+
+def main() -> None:
+    """Update the version of packages in the file ci/dpkg-versions.yaml."""
+
+    argparser = argparse.ArgumentParser(
+        description="Update the version of packages in the file ci/dpkg-versions.yaml."
+    )
+    argparser.add_argument("--branch", help="The branch to audit, not defined means autodetect")
+    args = argparser.parse_args()
+
+    cache: dict[str, dict[str, str]] = {}
+    yaml = ruamel.yaml.YAML()  # default_flow_style=False)
+    with open("ci/dpkg-versions.yaml", encoding="utf-8") as versions_file:
+        versions_config = yaml.load(versions_file)
+        for versions in versions_config.values():
+            for package_full in versions.keys():
+                dist, package = package_full.split("/")
+                if dist not in cache:
+                    correspondence = {
+                        "ubuntu_22_04": ("22.04", "jammy"),
+                    }
+                    if dist in correspondence:
+                        tag, dist_name = correspondence[dist]
+                        subprocess.run(
+                            ["docker", "rm", "--force", "apt"], stderr=subprocess.DEVNULL, check=False
+                        )
+                        subprocess.run(
+                            [
+                                "docker",
+                                "run",
+                                "--tty",
+                                "--interactive",
+                                "--detach",
+                                "--name=apt",
+                                "--entrypoint=",
+                                f"ubuntu:{tag}",
+                                "tail",
+                                "--follow",
+                                "/dev/null",
+                            ],
+                            check=True,
+                        )
+                        # Create a temporary file
+                        with tempfile.NamedTemporaryFile(mode="w", encoding=("utf-8")) as sources_list:
+                            sources_list.write(
+                                "\n".join(
+                                    [
+                                        f"deb http://archive.ubuntu.com/ubuntu/ {dist_name}-security main restricted",
+                                        f"deb http://archive.ubuntu.com/ubuntu/ {dist_name}-security universe",
+                                        f"deb http://archive.ubuntu.com/ubuntu/ {dist_name}-security multiverse",
+                                        "",
+                                    ]
+                                )
+                            )
+                            sources_list.flush()
+                            subprocess.run(
+                                ["docker", "cp", sources_list.name, "apt:/etc/apt/sources.list"], check=True
+                            )
+
+                        subprocess.run(["docker", "exec", "apt", "apt-get", "update"], check=True)
+
+                        package_re = re.compile(r"^([^ /]+)/[a-z-,]+ ([^ ]+) (all|amd64)( .*)?$")
+                        proc = subprocess.run(
+                            ["docker", "exec", "apt", "apt", "list"], check=True, stdout=subprocess.PIPE
+                        )
+                        for proc_line in proc.stdout.decode("utf-8").split("\n"):
+                            package_match = package_re.match(proc_line)
+                            if package_match is None:
+                                print(f"not matching: {proc_line}")
+                                continue
+                            cache.setdefault(dist, {})[package_match.group(1)] = package_match.group(2)
+
+                        subprocess.run(["docker", "rm", "--force", "apt"], check=True)
+
+                if package in cache[dist]:
+                    versions[package_full] = cache[dist][package]
+
+    with open("ci/dpkg-versions.yaml", "w", encoding="utf-8") as versions_file:
+        yaml.dump(versions_config, versions_file)
+
+    current_branch = c2cciutils.get_branch(args.branch)
+    c2cciutils.create_pull_request_if_needed(
+        current_branch, f"dpkg-update/{current_branch}", "Update dpkg package versions"
+    )
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -63,6 +63,7 @@ c2cciutils-docker-logs = "c2cciutils.scripts.docker_logs:main"
 c2cciutils-trigger-image-update = "c2cciutils.scripts.trigger_image_update:main"
 c2cciutils-download-applications = "c2cciutils.scripts.download_applications:main"
 c2cciutils-docker-versions-gen = "c2cciutils.scripts.docker_versions_gen:main"
+c2cciutils-docker-versions-update = "c2cciutils.scripts.docker_versions_update:main"
 
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"