From 10f3815b47146c3ab5403dd1ddde4c3ca6a330f4 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 6 May 2020 17:23:46 +0200 Subject: [PATCH 1/9] Fix optimization for materialized views --- graphite-ch-optimizer.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/graphite-ch-optimizer.go b/graphite-ch-optimizer.go index 6f68c79..1a6e4e0 100644 --- a/graphite-ch-optimizer.go +++ b/graphite-ch-optimizer.go @@ -23,10 +23,12 @@ import ( var version = "development" // SelectUnmerged is the query to create the temporary table with -// partitions and the retention age, which should be applied +// partitions and the retention age, which should be applied. +// Table name should be with backquotes to be able to OPTIMIZE `database`.`.inner.table` +// for MaterializedView engines const SelectUnmerged = ` SELECT - concat(p.database, '.', p.table) AS table, + concat(` + "'`', p.database, '`.`', p.table, '`'" + `) AS table, p.partition_id AS partition_id, p.partition AS partition_name, max(g.age) AS age, From 55fcd1454a436e04fe098b83d6231483d797d1cb Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 6 May 2020 18:26:39 +0200 Subject: [PATCH 2/9] Add tests action --- .github/workflows/tests.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..c52f251 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,35 @@ +name: Tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + tests: + name: Test code + runs-on: ubuntu-latest + strategy: + matrix: + go: + - ^1.13 + - ^1.14 + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Test + run: make test + + - name: Build and run version + run: | + make VERSION=testing-version + ./graphite-ch-optimizer --version From 21b59eaf0df1920cbb0ba862a189b626ff7140f2 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 May 2020 13:27:34 +0200 Subject: [PATCH 3/9] Add hash sums for packages --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3268b7a..6e073e9 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ define DESC = endef GO_FILES = $(shell find -name '*.go') PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm +SUM_FILES = build/sha256sum build/md5sum .PHONY: clean all version test @@ -37,7 +38,14 @@ build/$(NAME): $(NAME).go build/config.toml.example: build/$(NAME) ./build/$(NAME) --print-defaults > $@ -packages: $(PKG_FILES) +packages: $(PKG_FILES) $(SUM_FILES) + +$(SUM_FILES): COMMAND = $(notdir $@) +$(SUM_FILES): PKG_FILES_NAME = $(notdir $(PKG_FILES)) +.ONESHELL: +$(SUM_FILES): $(PKG_FILES) + cd build + $(COMMAND) $(PKG_FILES_NAME) > $(COMMAND) .ONESHELL: build/pkg: build/$(NAME) build/config.toml.example From b6e5fa53b3d1b90fc38795045a7a230b1d305bd8 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 May 2020 16:06:59 +0200 Subject: [PATCH 4/9] Refactor docker --- Makefile | 6 +++++- README.md | 4 ++++ docker/builder/Dockerfile | 8 ++++++++ .../graphite-ch-optimizer/Dockerfile | 12 +----------- 4 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 docker/builder/Dockerfile rename Dockerfile => docker/graphite-ch-optimizer/Dockerfile (69%) diff --git a/Makefile b/Makefile index 6e073e9..322052f 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ GO_FILES = $(shell find -name '*.go') PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm SUM_FILES = build/sha256sum build/md5sum -.PHONY: clean all version test +.PHONY: all clean docker test version all: build @@ -29,6 +29,10 @@ test: build: $(NAME) +docker: + docker build -t innogames/$(NAME):builder -f docker/builder/Dockerfile . + docker build -t innogames/$(NAME):latest -f docker/$(NAME)/Dockerfile . + $(NAME): $(NAME).go go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ . diff --git a/README.md b/README.md index 5e8b750..dc02c3f 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,10 @@ make rpm ``` ## Docker + +To build docker image locally run: +`make docker` + To launch the container run the following command on the host with a running ClickHouse server: `docker run --net=host --rm innogames/graphite-ch-optimizer:latest` diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile new file mode 100644 index 0000000..850e18e --- /dev/null +++ b/docker/builder/Dockerfile @@ -0,0 +1,8 @@ +# +# Image for building packages in docker +# innogames/graphite-ch-optimizer:builder on docker-hub +# +FROM golang:1.13-alpine as builder + +RUN apk --no-cache add ruby ruby-dev ruby-etc make gcc g++ rpm git tar && \ + gem install --no-user-install --no-document fpm diff --git a/Dockerfile b/docker/graphite-ch-optimizer/Dockerfile similarity index 69% rename from Dockerfile rename to docker/graphite-ch-optimizer/Dockerfile index 7ec044c..3ade606 100644 --- a/Dockerfile +++ b/docker/graphite-ch-optimizer/Dockerfile @@ -1,17 +1,7 @@ -# -# Image for building packages in docker -# innogames/graphite-ch-optimizer:builder on docker-hub -# -FROM golang:1.13-alpine as builder - -RUN apk --no-cache add ruby ruby-dev ruby-etc make gcc g++ rpm git tar && \ - gem install --no-user-install --no-document fpm - - # # Image which contains the binary artefacts # -FROM builder AS build +FROM innogames/graphite-ch-optimizer:builder AS build COPY . ./graphite-ch-optimizer WORKDIR ./graphite-ch-optimizer From 33f11dea269f3063eaea26978f6ad41f5a2216d0 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 May 2020 18:21:01 +0200 Subject: [PATCH 5/9] Add targets for github release --- .gitignore | 1 + Makefile | 51 ++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 3b4049e..55af516 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ graphite-ch-optimizer build/ +artifact/ diff --git a/Makefile b/Makefile index 322052f..997f343 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ GO_FILES = $(shell find -name '*.go') PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm SUM_FILES = build/sha256sum build/md5sum + .PHONY: all clean docker test version all: build @@ -18,16 +19,19 @@ version: @echo $(VERSION) clean: + rm -rf artifact rm -rf build rm -rf $(NAME) rebuild: clean all +# Run tests test: go vet $(GO_FILES) go test $(GO_FILES) -build: $(NAME) +build: | $(NAME) + mkdir build docker: docker build -t innogames/$(NAME):builder -f docker/builder/Dockerfile . @@ -36,6 +40,38 @@ docker: $(NAME): $(NAME).go go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ . +######################################################### +# Prepare artifact directory and set outputs for upload # +######################################################### +github_artifact: $(foreach art,$(PKG_FILES) $(SUM_FILES), artifact/$(notdir $(art))) + +artifact: + mkdir $@ + +# Link artifact to directory with setting step output to filename +artifact/%: ART=$(notdir $@) +artifact/%: TYPE=$(lastword $(subst ., ,$(ART))) +artifact/%: build/% | artifact + cp -l $< $@ + @echo '::set-output name=$(TYPE)::$(ART)' + +####### +# END # +####### + +############# +# Packaging # +############# + +# Prepare everything for packaging +.ONESHELL: +build/pkg: build/$(NAME) build/config.toml.example + cd build + mkdir -p pkg/etc/$(NAME) + mkdir -p pkg/usr/bin + cp -l $(NAME) pkg/usr/bin/ + cp -l config.toml.example pkg/etc/$(NAME) + build/$(NAME): $(NAME).go GOOS=linux GOARCH=amd64 go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ . @@ -44,6 +80,7 @@ build/config.toml.example: build/$(NAME) packages: $(PKG_FILES) $(SUM_FILES) +# md5 and sha256 sum-files for packages $(SUM_FILES): COMMAND = $(notdir $@) $(SUM_FILES): PKG_FILES_NAME = $(notdir $(PKG_FILES)) .ONESHELL: @@ -51,14 +88,6 @@ $(SUM_FILES): $(PKG_FILES) cd build $(COMMAND) $(PKG_FILES_NAME) > $(COMMAND) -.ONESHELL: -build/pkg: build/$(NAME) build/config.toml.example - cd build - mkdir -p pkg/etc/$(NAME) - mkdir -p pkg/usr/bin - cp -l $(NAME) pkg/usr/bin/ - cp -l config.toml.example pkg/etc/$(NAME) - deb: $(word 1, $(PKG_FILES)) rpm: $(word 2, $(PKG_FILES)) @@ -82,3 +111,7 @@ $(PKG_FILES): build/pkg -p build \ build/pkg/=/ \ packaging/$(NAME).service=/lib/systemd/system/$(NAME).service + +####### +# END # +####### From 7de4c61c346b620616112f1f66d1a5a0ba161752 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 May 2020 18:45:41 +0200 Subject: [PATCH 6/9] Implement build and upload assets workflow --- .github/workflows/upload-assets.yml | 80 +++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/upload-assets.yml diff --git a/.github/workflows/upload-assets.yml b/.github/workflows/upload-assets.yml new file mode 100644 index 0000000..fb695ab --- /dev/null +++ b/.github/workflows/upload-assets.yml @@ -0,0 +1,80 @@ +name: Upload Packages to new release + +on: + release: + types: + - published + +jobs: + build: + name: Build + runs-on: ubuntu-latest + + container: + image: innogames/graphite-ch-optimizer:builder + + outputs: + deb: ${{ steps.build.outputs.deb }} + rpm: ${{ steps.build.outputs.rpm }} + sha256sum: ${{ steps.build.outputs.sha256sum }} + md5sum: ${{ steps.build.outputs.md5sum }} + + steps: + - uses: actions/checkout@v2 + name: Checkout + with: + # Otherwise there's a risk to not get latest tag + # We hope, that the current commit at + # least 50 commits close to the latest release + fetch-depth: 50 + - name: Build packages + id: build + run: | + # Checkout action doesn't fetch tags + git fetch --tags + make -e CGO_ENABLED=0 packages + make github_artifact + - name: Upload rpm + id: upload-rpm + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ASSET: ${{ steps.build.outputs.rpm }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: artifact/${{ env.ASSET }} + asset_name: ${{ env.ASSET }} + asset_content_type: application/octet-stream + - name: Upload sha256sum + id: upload-sha256sum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ASSET: ${{ steps.build.outputs.sha256sum }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: artifact/${{ env.ASSET }} + asset_name: ${{ env.ASSET }} + asset_content_type: application/octet-stream + - name: Upload md5sum + id: upload-md5sum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ASSET: ${{ steps.build.outputs.md5sum }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: artifact/${{ env.ASSET }} + asset_name: ${{ env.ASSET }} + asset_content_type: application/octet-stream + - name: Upload deb + id: upload-deb + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ASSET: ${{ steps.build.outputs.deb }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: artifact/${{ env.ASSET }} + asset_name: ${{ env.ASSET }} + asset_content_type: application/octet-stream From 5190ff353a44db1cf3c89fa626be559dc6649576 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 8 May 2020 16:17:08 +0200 Subject: [PATCH 7/9] Rework jenkins job to download package from release --- Jenkinsfile | 64 +++++++++-------------------------------------------- 1 file changed, 10 insertions(+), 54 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ee83330..cbd4289 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,34 +1,20 @@ #!/usr/bin/groovy library 'adminsLib@master' -import groovy.json.JsonOutput properties([ parameters([ string(defaultValue: '', description: 'deb-drop repository, see https://github.com/innogames/deb-drop/', name: 'REPO_NAME', trim: false), - booleanParam(defaultValue: false, description: 'Publish new packages and images or not. True for new releases, False by default.', name: 'PUBLISH'), ]) ]) -def build_packages(docker_image) { - env.IMAGE = docker_image.imageName() - sh ''' - #!/bin/bash - docker run --rm "${IMAGE}" | tar x --wildcards 'build/graphite-ch-optimizer?*' - ''' -} - - // Remove builds in presented status, default is ['ABORTED', 'NOT_BUILT'] jobCommon.cleanNotFinishedBuilds() -node('docker') { +node() { ansiColor('xterm') { // Checkout repo and get info about current stage sh 'echo Initial env; env | sort' env.PACKAGE_NAME = 'graphite-ch-optimizer' - env.DOCKER_IMAGE = 'innogames/' + env.PACKAGE_NAME - def img_builder - def img_build try { stage('Checkout') { gitSteps checkout: true, changeBuildName: false @@ -36,49 +22,19 @@ ansiColor('xterm') { env.NEW_VERSION = sh(returnStdout: true, script: 'make version').trim() currentBuild.displayName = "${currentBuild.number}: version ${env.NEW_VERSION}" } - stage('Tests') { - try { - docker.image("${env.DOCKER_IMAGE}:builder").pull() - } catch (all) { - echo 'Unable to pull builder image, building from scratch' - } - img_builder = docker.build( - "${env.DOCKER_IMAGE}:builder", - "--target builder --cache-from=${env.DOCKER_IMAGE}:builder ." - ) - // Make test is the part of the build image - img_build = docker.build("${env.DOCKER_IMAGE}:build", '--target build .') - if (env.GIT_BRANCH == 'master') { - docker.withRegistry('', 'dockerIgSysadminsToken') { - img_builder.push() + stage('Upload to deb-drop') { + when(env.GIT_BRANCH_OR_TAG == 'tag' && jobCommon.launchedByUser() && env.REPO_NAME != '') { + deb_package = "graphite-ch-optimizer_${env.NEW_VERSION}_amd64.deb" + [deb_package, 'md5sum', 'sha256sum'].each { file-> + sh "set -ex; wget https://github.com/innogames/graphite-ch-optimizer/releases/download/${env.GIT_BRANCH}/${file}" + } + ['md5sum', 'sha256sum'].each { sum-> + sh "set -ex; ${sum} --ignore-missing --status -c ${sum}" } - } - } - stage('Building') { - when(jobCommon.launchedByUser()) { - build_packages(img_build) - - if (env.REPO_NAME) { - deb_packages = findFiles(glob: "build/*${env.NEW_VERSION}*deb") withCredentials([string(credentialsId: 'DEB_DROP_TOKEN', variable: 'DebDropToken')]) { - deb_packages.each { pack-> - jobCommon.uploadPackage file: pack.path, repo: env.REPO_NAME, token: DebDropToken - } + jobCommon.uploadPackage file: deb_package, repo: env.REPO_NAME, token: DebDropToken } } - - } - when(env.GIT_BRANCH_OR_TAG == 'tag' && !jobCommon.launchedByUser()) { - // TODO: implement github-api requests in library to publish new releases and assets - echo 'Assets publishing will be here eventually' - //build_packages(img_build) - //withCredentials([usernamePassword( - // credentialsId: 'username/token', - // usernameVariable: 'USERNAME', - // passwordVariable: 'TOKEN' - //)]) { - //} - } } cleanWs(notFailBuild: true) } From 7d65cfa777ac0f7690f5b8b9497bd182b979d848 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 14 May 2020 13:39:38 +0200 Subject: [PATCH 8/9] Replace wrong read_timeout parameter with the proper --- README.md | 6 +++--- graphite-ch-optimizer.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index dc02c3f..d481ce4 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ To launch the container run the following command on the host with a running Cli * Daemon mode is preferable over one-shot script for the normal work * It's safe to run it on the cluster hosts * You could either run it on the one of replicated host or just over the all hosts -* If you have big partitions (month or something like this) and will get exceptions about timeout, then you need to adjust `read_timeout` parameter in DSN +* If you have big partitions (month or something like this) and will get exceptions about timeout, then you need to adjust `receive_timeout` parameter in DSN * `optimize_throw_if_noop=1` is not mandatory, but good to have. * The next picture demonstrates the result of running the daemon for the first time on ~3 years old GraphiteMergeTree table: example @@ -152,7 +152,7 @@ Default config: ```toml [clickhouse] optimize-interval = "72h0m0s" - server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&read_timeout=3600&debug=true" + server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true" [daemon] dry-run = false @@ -172,7 +172,7 @@ Usage of graphite-ch-optimizer: --print-defaults Print default config values and exit -v, --version Print version and exit --optimize-interval duration The active partitions won't be optimized more than once per this interval, seconds (default 72h0m0s) - -s, --server-dsn string DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&read_timeout=3600&debug=true") + -s, --server-dsn string DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true") -n, --dry-run Will print how many partitions would be merged without actions --loop-interval duration Daemon will check if there partitions to merge once per this interval, seconds (default 1h0m0s) --one-shot Program will make only one optimization instead of working in the loop (true if dry-run) diff --git a/graphite-ch-optimizer.go b/graphite-ch-optimizer.go index 1a6e4e0..807cda0 100644 --- a/graphite-ch-optimizer.go +++ b/graphite-ch-optimizer.go @@ -137,7 +137,7 @@ func init() { func setDefaultConfig() { viper.SetDefault("clickhouse", map[string]interface{}{ // See ClickHouse documentation for further options - "server-dsn": "tcp://localhost:9000?&optimize_throw_if_noop=1&read_timeout=3600&debug=true", + "server-dsn": "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true", // Ignore partitions which were merged less than 3 days before "optimize-interval": time.Duration(72) * time.Hour, }) From 63cf6058ac8da1950a7c150491b92f2cb8d03792 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 14 May 2020 14:23:58 +0200 Subject: [PATCH 9/9] Add docs for releases and docker hub images --- docs/DOCKER.md | 6 ++++++ docs/RELEASE.md | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 docs/DOCKER.md create mode 100644 docs/RELEASE.md diff --git a/docs/DOCKER.md b/docs/DOCKER.md new file mode 100644 index 0000000..185575b --- /dev/null +++ b/docs/DOCKER.md @@ -0,0 +1,6 @@ +# Docker images and tags +The images `innogames/graphite-ch-optimizer` are available on docker-hub with the following tags: + +- `${semantic_version}` (`docker/graphite-ch-optimizer/Dockerfile`) - these tags are automatically built from tags with tag regexp `v([.0-9]+)` +- `latest` (`docker/graphite-ch-optimizer/Dockerfile`) - is built automatically from the latest `master` commit +- `builder` (`docker/builder/Dockerfile`) - is built automatically from the latest `master` commit diff --git a/docs/RELEASE.md b/docs/RELEASE.md new file mode 100644 index 0000000..e8f476d --- /dev/null +++ b/docs/RELEASE.md @@ -0,0 +1,6 @@ +# CREATE NEW RELEASE +When the code is ready to new release, create it with the tag `v${major_version}.${minor_version}.${patch_version}`. +We have a [workflow](../.github/workflows/upload-assets.yml), which will upload created DEB and RPM packages together with hash sums as the release assets. + +## Use Jenkins to upload packages to deb-drop repository +When the release is ready and assets are uploaded, launch the multibranch pipeline job configured against [Jenkinsfile](../Jenkinsfile) with desired version. It will download the package, compare hashsums and upload it to the repository.