Merge branch 'customizations/24.3.12' into backports/24.3/72190-prefe…

…r_locahost_replica-in-parallel_distributed_insert_select

.github/workflows/reusable_build.yml

@@ -54,7 +54,7 @@ jobs:
     if: ${{ contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name) || inputs.force }}
     env:
       GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}}
-    runs-on: [self-hosted, altinity-type-ccx53, altinity-on-demand, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
     steps:
       - name: Check out repository code
         uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6

.github/workflows/reusable_docker.yml

@@ -29,7 +29,7 @@ env:
 
 jobs:
   DockerBuildAarch64:
-    runs-on: [altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none]
+    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
     if: |
       !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_aarch64) != '[]'
     steps:
@@ -44,7 +44,7 @@ jobs:
             --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
             --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_aarch64) }}'
   DockerBuildAmd64:
-    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
+    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
     if: |
       !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_amd64) != '[]'
     steps:
@@ -60,7 +60,7 @@ jobs:
             --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_amd64) }}'
   DockerMultiArchManifest:
     needs: [DockerBuildAmd64, DockerBuildAarch64]
-    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
+    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
     if: |
       !failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest)
     steps:

.github/workflows/reusable_sign.yml

@@ -76,7 +76,7 @@ jobs:
         run: |
 
           # Prepend self-hosted
-          input="self-hosted, ${input}"
+          input="self-hosted, altinity-on-demand, ${input}"
 
           # Remove all whitespace
           input="$(echo ${input} | tr -d [:space:])"

.github/workflows/reusable_test.yml

@@ -70,7 +70,7 @@ jobs:
         run: |
 
           # Prepend self-hosted
-          input="self-hosted, ${input}"
+          input="self-hosted, altinity-on-demand, ${input}"
 
           # Remove all whitespace
           input="$(echo ${input} | tr -d [:space:])"

cmake/autogenerated_versions.txt

@@ -8,9 +8,11 @@ SET(VERSION_MINOR 3)
 SET(VERSION_PATCH 12)
 SET(VERSION_GITHASH 28795d0a47ebd38dc9a6b82038074eb0352b0f25)
 
-SET(VERSION_TWEAK 76)
+# This is the 'base' tweak of the version, build scripts will
+# increment this by number of commits since previous tag.
+SET(VERSION_TWEAK 10000)
 SET(VERSION_FLAVOUR altinitystable)
 
-SET(VERSION_DESCRIBE v24.3.12.76.altinitystable)
-SET(VERSION_STRING 24.3.12.76.altinitystable)
+SET(VERSION_DESCRIBE v24.3.12.10000.altinitystable)
+SET(VERSION_STRING 24.3.12.10000.altinitystable)
 # end of autochange

rust/skim/Cargo.toml

@@ -7,11 +7,11 @@ edition = "2021"
 
 [dependencies]
 skim = { version = "0.10.2", default-features = false }
-cxx = "1.0.83"
+cxx = "=1.0.83"
 term = "0.7.0"
 
 [build-dependencies]
-cxx-build = "1.0.83"
+cxx-build = "=1.0.83"
 
 [lib]
 crate-type = ["staticlib"]

tests/ci/build_check.py

@@ -20,13 +20,14 @@
     TEMP_PATH,
 )
 from git_helper import Git
-from pr_info import PRInfo
+from pr_info import PRInfo, EventType
 from report import FAILURE, JobReport, StatusType, SUCCESS
 from s3_helper import S3Helper
 from tee_popen import TeePopen
 import docker_images_helper
 from version_helper import (
     ClickHouseVersion,
+    VersionType,
     get_version_from_repo,
     update_version_local,
 )
@@ -174,16 +175,30 @@ def main():
     version = get_version_from_repo(git=Git(True))
     logging.info("Got version from repo %s", version.string)
 
-    official_flag = pr_info.number == 0
+    # official_flag = pr_info.number == 0
 
-    version_type = "testing"
-    if "release" in pr_info.labels or "release-lts" in pr_info.labels:
-        version_type = "stable"
-        official_flag = True
+    # version_type = "testing"
+    # if "release" in pr_info.labels or "release-lts" in pr_info.labels:
+    #     version_type = "stable"
+    #     official_flag = True
+
+    # NOTE(vnemkov): For Altinity Stable builds, version flavor
+    #  (last part of version, like 'altinitystable') is obtained from tag.
+    # If there is no tag, then version is considered to be 'testing'
+    version_type = version._flavour = VersionType.TESTING
+    official_flag = True
+
+    if pr_info.event_type == EventType.PUSH \
+            and pr_info.ref.startswith('refs/tags/'):
+        tag_name = pr_info.ref.removeprefix('refs/tags/')
+        version_type = tag_name.split('.')[-1]
+        version._flavour = version_type
+        logging.info("Using version from tag: %s => %s", tag_name, version)
 
+    # NOTE(vnemkov): make sure that version is set correctly for the generated binaries
     update_version_local(version, version_type)
 
-    logging.info("Updated local files with version")
+    logging.info(f"Updated local files with version : {version.string} / {version.describe}")
 
     logging.info("Build short name %s", build_name)
 

tests/ci/build_report_check.py

@@ -78,9 +78,9 @@ def main():
             missing_builds += 1
             build_results.insert(0, build_result)
         else:
-            assert (
-                pr_info.head_ref == build_result.head_ref or pr_info.number > 0
-            ), "BUG. if not a PR, report must be created on the same branch"
+            # if pr_info.head_ref == build_result.head_ref or pr_info.number > 0:
+            logging.error("BUG. if not a PR, report must be created on the same branch, build_name: %s, \npr_info: %s,\nbuild_result: %s",
+                build_name, pr_info, build_result)
             build_results.append(build_result)
 
     # Calculate artifact groups like packages and binaries

tests/ci/ci.py

@@ -53,7 +53,11 @@
 from pr_info import PRInfo
 from report import ERROR, SUCCESS, BuildResult, JobReport
 from s3_helper import S3Helper
-from version_helper import get_version_from_repo
+from version_helper import (
+    get_version_from_repo,
+    get_version_from_string,
+    update_cmake_version
+)
 
 # pylint: disable=too-many-lines
 
@@ -1947,6 +1951,20 @@ def main() -> int:
         print(
             f"Check if rerun for name: [{check_name}], extended name [{check_name_with_group}]"
         )
+
+        # NOTE (vnemkov) Job might have not checked out git tags, so it can't properly compute version number.
+        # BUT if there is pre-computed version from `RunConfig` then we can reuse it.
+        pre_configured_version = indata.get('version', None)
+        git = Git(True)
+        if pre_configured_version is not None and git.commits_since_latest == 0:
+            print(f"Updating version in repo files from '{get_version_from_repo()}' to '{pre_configured_version}'")
+
+            pre_configured_version = get_version_from_string(pre_configured_version, git)
+            # need to set description, otherwise subsequent call (perhaps from other script) to get_version_from_repo() fails
+            pre_configured_version.with_description(pre_configured_version.flavour)
+
+            update_cmake_version(pre_configured_version)
+
         previous_status = None
         if CI_CONFIG.is_build_job(check_name):
             # this is a build job - check if build report is present

tests/ci/git_helper.py

@@ -9,6 +9,22 @@
 
 logger = logging.getLogger(__name__)
 
+class VersionType:
+    LTS = "lts"
+    NEW = "new"
+    PRESTABLE = "altinityedge"
+    STABLE = "altinitystable"
+    TESTING = "altinitytest"
+    ANTALYA = "altinityantalya"
+    HOTFIX = "altinityhotfix"
+
+    VALID = (NEW, TESTING, HOTFIX, PRESTABLE, STABLE, LTS, ANTALYA,
+            # NOTE (vnemkov): we don't use those directly, but it is used in unit-tests
+            "stable",
+            "prestable",
+            "testing",
+    )
+
 # ^ and $ match subline in `multiple\nlines`
 # \A and \Z match only start and end of the whole string
 # NOTE (vnemkov): support both upstream tag style: v22.x.y.z-lts and Altinity tag style: v22.x.y.z.altinitystable
@@ -17,8 +33,9 @@
 TAG_REGEXP = (
     r"\Av\d{2}"  # First two digits of major part
     r"([.][1-9]\d*){3}"  # minor.patch.tweak parts
-    r"-(new|testing|prestable|stable|lts|altinitystable)\Z"  # suffix with a version type
+    fr"[.-]({'|'.join(VersionType.VALID)})\Z"  # suffix with a version type
 )
+
 SHA_REGEXP = re.compile(r"\A([0-9]|[a-f]){40}\Z")
 
 CWD = p.dirname(p.realpath(__file__))
@@ -171,7 +188,7 @@ def _update_tags(self, suppress_stderr: bool = False) -> None:
         stderr = subprocess.DEVNULL if suppress_stderr else None
         self.latest_tag = self.run("git describe --tags --abbrev=0", stderr=stderr)
         # Format should be: {latest_tag}-{commits_since_tag}-g{sha_short}
-        self.commits_since_latest = int(
+        self.commits_since_latest = self.commits_since_tag = int(
             self.run(f"git rev-list {self.latest_tag}..HEAD --count")
         )
         if self.latest_tag.endswith("-new"):

tests/ci/pr_info.py

@@ -71,11 +71,9 @@ def get_pr_for_commit(sha, ref):
             if pr["head"]["ref"] in ref:
                 return pr
             our_prs.append(pr)
-        print(
-            f"Cannot find PR with required ref {ref}, sha {sha} - returning first one"
-        )
-        first_pr = our_prs[0]
-        return first_pr
+        logging.warning("Cannot find PR with required ref %s, sha %s", ref, sha)
+        # first_pr = our_prs[0]
+        return None
     except Exception as ex:
         print(f"Cannot fetch PR info from commit {ref}, {sha}", ex)
     return None
@@ -121,6 +119,14 @@ def __init__(
         ref = github_event.get("ref", "refs/heads/master")
         if ref and ref.startswith("refs/heads/"):
             ref = ref[11:]
+        self.ref : str = ref # e.g. "refs/pull/509/merge" or "refs/tags/v24.3.12.76.altinitystable"
+
+        # Default values
+        self.base_ref = github_event.get("base_ref","")  # type: str
+        self.base_name = ""  # type: str
+        self.head_ref = ""  # type: str
+        self.head_name = ""  # type: str
+        self.number = 0  # type: int
 
         # workflow completed event, used for PRs only
         if "action" in github_event and github_event["action"] == "completed":
@@ -163,15 +169,11 @@ def __init__(
             # master or backport/xx.x/xxxxx - where the PR will be merged
             self.base_ref = github_event["pull_request"]["base"]["ref"]  # type: str
             # ClickHouse/ClickHouse
-            self.base_name = github_event["pull_request"]["base"]["repo"][
-                "full_name"
-            ]  # type: str
+            self.base_name = github_event["pull_request"]["base"]["repo"]["full_name"]  # type: str
             # any_branch-name - the name of working branch name
             self.head_ref = github_event["pull_request"]["head"]["ref"]  # type: str
             # UserName/ClickHouse or ClickHouse/ClickHouse
-            self.head_name = github_event["pull_request"]["head"]["repo"][
-                "full_name"
-            ]  # type: str
+            self.head_name = github_event["pull_request"]["head"]["repo"]["full_name"]  # type: str
             self.body = github_event["pull_request"]["body"]
             self.labels = {
                 label["name"] for label in github_event["pull_request"]["labels"]
@@ -228,17 +230,22 @@ def __init__(
             self.commit_html_url = f"{repo_prefix}/commit/{self.sha}"
 
             if pull_request is None or pull_request["state"] == "closed":
-                # it's merged PR to master
+                # it's merged PR to master, or there is no PR (build against specific commit or tag)
                 self.number = 0
                 self.labels = set()
                 self.pr_html_url = f"{repo_prefix}/commits/{ref}"
                 self.base_ref = ref
                 self.base_name = self.repo_full_name
                 self.head_ref = ref
                 self.head_name = self.repo_full_name
-                self.diff_urls.append(
-                    self.compare_url(github_event["before"], self.sha)
-                )
+
+                before_sha = github_event["before"]
+                # in case of just a tag on existing commit, "before_sha" is 0000000000000000000000000000000000000000
+                # Hence it is a special case and basically nothing changed, there is no need to compose a diff url
+                if not all(x == '0' for x in before_sha):
+                    self.diff_urls.append(
+                        self.compare_url(before_sha, self.sha)
+                    )
             else:
                 self.number = pull_request["number"]
                 self.labels = {label["name"] for label in pull_request["labels"]}