Update/Reload without downtime

.github/workflows/apt-arm.yml

@@ -1,6 +1,9 @@
 name: Apt based Linux (AArch64)
 on:
   workflow_dispatch:
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
 jobs:
   build:
     name: Build

.github/workflows/apt.yml

@@ -5,10 +5,13 @@ on:
       - master
       - fluent-package-lts-v5
   pull_request:
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
 jobs:
   build:
     name: Build
-    timeout-minutes: 60
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -51,16 +54,34 @@ jobs:
             fluent-package/apt/repositories
             fluent-apt-source/apt/repositories
             fluent-lts-apt-source/apt/repositories
+            v6-test/fluent-package/apt/repositories
           key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', 'fluent-package/templates/**', 'fluent-package/debian/**', 'fluent-package/apt/**/Dockerfile') }}
       - name: Build deb with Docker
         if: ${{ ! steps.cache-deb.outputs.cache-hit }}
         run: |
           rake apt:build APT_TARGETS=${{ matrix.rake-job }}
+      - uses: actions/checkout@master
+        if: ${{ ! steps.cache-deb.outputs.cache-hit }}
+        with:
+          path: v6-test
+      - name: Build v6 deb with Docker
+        if: ${{ ! steps.cache-deb.outputs.cache-hit }}
+        run: |
+          cd v6-test
+          git config user.email "[email protected]"
+          git config user.name "Fluentd developers"
+          git am fluent-package/bump-version-v6.patch
+          rake apt:build APT_TARGETS=${{ matrix.rake-job }}
       - name: Upload fluent-package deb
         uses: actions/upload-artifact@master
         with:
           name: packages-${{ matrix.rake-job }}
           path: fluent-package/apt/repositories
+      - name: Upload v6 fluent-package deb
+        uses: actions/upload-artifact@master
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test/fluent-package/apt/repositories
       - name: Upload fluent-apt-source deb
         uses: actions/upload-artifact@master
         with:
@@ -154,6 +175,10 @@ jobs:
           - "update-to-next-version-service-status.sh enabled inactive"
           - "update-to-next-version-service-status.sh disabled active"
           - "update-to-next-version-service-status.sh disabled inactive"
+          - "update-to-next-version-with-auto-and-manual.sh"
+          - "update-to-next-major-version.sh"
+          - "update-without-data-lost.sh v5 v6"
+          - "update-without-data-lost.sh v6 v5"
         include:
           - label: Debian bullseye amd64
             rake-job: debian-bullseye
@@ -196,9 +221,31 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test
       - uses: actions/download-artifact@v4
         with:
           name: packages-apt-source-${{ matrix.rake-job }}
       - uses: canonical/[email protected]
+      - name: Run diagnostic
+        run: |
+          uname -a
+          echo "::group::snap info lxd"
+          snap info lxd
+          echo "::endgroup::"
+          echo "::group::snap services lxd"
+          snap services lxd
+          echo "::endgroup::"
+          echo "::group::snap logs lxd"
+          sudo snap logs lxd
+          echo "::endgroup::"
+          echo "::group::lxc remote list"
+          lxc remote list
+          echo "::endgroup::"
+          echo "::group::lxc list images:"
+          lxc image list images:
+          echo "::endgroup::"
       - name: Run Test  ${{ matrix.test }} on ${{ matrix.lxc-image }}
         run: fluent-package/apt/systemd-test/test.sh ${{ matrix.lxc-image }} ${{ matrix.test }}

.github/workflows/macos.yml

@@ -5,6 +5,9 @@ on:
       - master
       - fluent-package-lts-v5
   pull_request:
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
 jobs:
   build:
     name: Build

.github/workflows/windows.yml

@@ -5,6 +5,9 @@ on:
       - master
       - fluent-package-lts-v5
   pull_request:
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
 jobs:
   build:
     name: Build

.github/workflows/yum-arm.yml

@@ -1,6 +1,9 @@
 name: Yum based Linux (AArch64)
 on:
   workflow_dispatch:
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
 jobs:
   build:
     name: Build

.github/workflows/yum.yml

@@ -5,10 +5,13 @@ on:
       - master
       - fluent-package-lts-v5
   pull_request:
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
 jobs:
   build:
     name: Build
-    timeout-minutes: 60
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -47,17 +50,36 @@ jobs:
         uses: actions/cache@v4
         id: cache-rpm
         with:
-          path: fluent-package/yum/repositories
+          path: |
+            fluent-package/yum/repositories
+            v6-test/fluent-package/yum/repositories
           key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', '**/*.spec.in', 'fluent-package/templates/**', 'fluent-package/yum/**/Dockerfile') }}
       - name: Build rpm with Docker
         if: ${{ ! steps.cache-rpm.outputs.cache-hit }}
         run: |
           rake yum:build YUM_TARGETS=${{ matrix.rake-job }}
+      - uses: actions/checkout@master
+        if: ${{ ! steps.cache-rpm.outputs.cache-hit }}
+        with:
+          path: v6-test
+      - name: Build v6 rpm with Docker
+        if: ${{ ! steps.cache-rpm.outputs.cache-hit }}
+        run: |
+          cd v6-test
+          git config user.email "[email protected]"
+          git config user.name "Fluentd developers"
+          git am fluent-package/bump-version-v6.patch
+          rake yum:build YUM_TARGETS=${{ matrix.rake-job }}
       - name: Upload fluent-package rpm
         uses: actions/upload-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
           path: fluent-package/yum/repositories
+      - name: Upload v6 fluent-package rpm
+        uses: actions/upload-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test/fluent-package/yum/repositories
       # TODO move the following steps to "Test" job
       - name: Check Package Size
         run: |
@@ -128,6 +150,10 @@ jobs:
           - "update-to-next-version-service-status.sh enabled inactive"
           - "update-to-next-version-service-status.sh disabled active"
           - "update-to-next-version-service-status.sh disabled inactive"
+          - "update-to-next-version-with-auto-and-manual.sh"
+          - "update-to-next-major-version.sh"
+          - "update-without-data-lost.sh v5 v6"
+          - "update-without-data-lost.sh v6 v5"
         include:
           - label: AmazonLinux 2 x86_64
             rake-job: amazonlinux-2
@@ -147,7 +173,29 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test
       - uses: canonical/[email protected]
+      - name: Run diagnostic
+        run: |
+          uname -a
+          echo "::group::snap info lxd"
+          snap info lxd
+          echo "::endgroup::"
+          echo "::group::snap services lxd"
+          snap services lxd
+          echo "::endgroup::"
+          echo "::group::snap logs lxd"
+          sudo snap logs lxd
+          echo "::endgroup::"
+          echo "::group::lxc remote list"
+          lxc remote list
+          echo "::endgroup::"
+          echo "::group::lxc list images:"
+          lxc image list images:
+          echo "::endgroup::"
       - name: Run Test ${{ matrix.test }} on ${{ matrix.lxc-image }}
         run: fluent-package/yum/systemd-test/test.sh ${{ matrix.lxc-image }} ${{ matrix.test }}
 
@@ -174,6 +222,10 @@ jobs:
           - "update-to-next-version-service-status.sh enabled inactive"
           - "update-to-next-version-service-status.sh disabled active"
           - "update-to-next-version-service-status.sh disabled inactive"
+          - "update-to-next-version-with-auto-and-manual.sh"
+          - "update-to-next-major-version.sh"
+          - "update-without-data-lost.sh v5 v6"
+          - "update-without-data-lost.sh v6 v5"
         include:
           - label: RockyLinux 8 x86_64
             rake-job: rockylinux-8
@@ -186,6 +238,10 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test
       - uses: canonical/[email protected]
       - name: Run Test ${{ matrix.test }} on ${{ matrix.lxc-image }}
         run: fluent-package/yum/systemd-test/test.sh ${{ matrix.lxc-image }} ${{ matrix.test }}

fluent-package/Gemfile

@@ -13,7 +13,7 @@ gem "cool.io", "1.8.1",  platforms: no_fat_gem
 gem "sigdump", "0.2.5"
 gem "http_parser.rb", "0.8.0"
 gem "yajl-ruby", "1.4.3"
-gem "serverengine", '2.3.2'
+gem "serverengine", '2.4.0'
 gem "msgpack", "1.7.2"
 gem "oj", "3.16.4"
 gem "tzinfo", "2.0.6"

fluent-package/Gemfile.lock

@@ -9,17 +9,18 @@ GIT
 
 GIT
   remote: https://github.com/fluent/fluentd
-  revision: e763c0761c44d9734b6aa374371387a2e8406522
-  ref: e763c0761c44d9734b6aa374371387a2e8406522
+  revision: f8755d09b747d054302c0e12a3b0962a11f1045e
+  ref: f8755d09b747d054302c0e12a3b0962a11f1045e
   specs:
-    fluentd (1.17.0)
+    fluentd (1.17.1)
       base64 (~> 0.2)
       bundler
       certstore_c (~> 0.1.7)
       cool.io (>= 1.4.5, < 2.0.0)
       csv (~> 3.2)
       drb (~> 2.2)
       http_parser.rb (>= 0.5.1, < 0.9.0)
+      logger (~> 1.6)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.3.2, < 3.0.0)
       sigdump (~> 0.2.5)
@@ -236,7 +237,9 @@ GEM
       digest-crc
     ruby-progressbar (1.13.0)
     rubyzip (1.3.0)
-    serverengine (2.3.2)
+    serverengine (2.4.0)
+      base64 (~> 0.1)
+      logger (~> 1.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.5)
     strptime (0.2.5)
@@ -337,7 +340,7 @@ DEPENDENCIES
   rake
   rdkafka (= 0.16.1)
   ruby-kafka (= 1.5.0)
-  serverengine (= 2.3.2)
+  serverengine (= 2.4.0)
   sigdump (= 0.2.5)
   systemd-journal (= 1.4.2)
   td (= 0.17.1)

fluent-package/Rakefile

@@ -157,7 +157,7 @@ def extract_fluentd_version(archive_path)
                           File.basename(archive_path, ".tar.gz"))
   Dir.chdir(archive_dir) do
     IO.popen("git tag --contains #{FLUENTD_REVISION}") do |tags|
-      fluentd_version = tags.readlines.last.chomp.delete_prefix("v")
+      fluentd_version = tags.readlines.last&.chomp&.delete_prefix("v") || ""
     end
   end
   puts "::endgroup::" if ENV["CI"]
@@ -1183,10 +1183,21 @@ class BuildTask
     fluentd_conf = "etc/#{PACKAGE_DIR}/#{SERVICE_NAME}.conf"
     fluentd_conf_default = "opt/#{PACKAGE_DIR}/share/#{SERVICE_NAME}.conf"
     configs = [fluentd_conf]
-    configs.concat([
-                     "etc/logrotate.d/#{SERVICE_NAME}",
-                     fluentd_conf_default,
-                   ]) unless windows? || macos?
+    unless windows? || macos?
+      configs.concat([
+                      "etc/logrotate.d/#{SERVICE_NAME}",
+                      fluentd_conf_default,
+                    ])
+      File.readlines("/etc/os-release").each do |line|
+        if line.include?("ID=")
+          distribution = line.split(/=/).last.chomp
+          if ["debian", "ubuntu"].include?(distribution)
+            puts "Suppress needrestart on #{distribution}"
+            configs.concat(["etc/needrestart/conf.d/50-fluent-package.conf"])
+          end
+        end
+      end
+    end
     configs.each do |config|
       src =
         if config == fluentd_conf_default

fluent-package/apt/systemd-test/test.sh

@@ -15,7 +15,7 @@ dir="/host/fluent-package/apt/systemd-test"
 set -eux
 
 echo "::group::Run test: launch $image"
-lxc launch $image target
+lxc launch $image target --debug
 sleep 5
 echo "::endgroup::"
 echo "::group::Run test: configure $image"

fluent-package/apt/systemd-test/update-from-v4.sh

@@ -32,7 +32,19 @@ case $1 in
   local)
     sudo apt install -V -y \
       /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb \
-      /host/${distribution}/pool/${code_name}/${channel}/*/*/td-agent_*_all.deb
+      /host/${distribution}/pool/${code_name}/${channel}/*/*/td-agent_*_all.deb 2>&1 | tee upgrade.log
+    # Test: needrestart was suppressed
+    if dpkg-query --show --showformat='${Version}' needrestart ; then
+      case $code_name in
+        focal)
+          # dpkg-query succeeds even though needrestart is not installed.
+          (! grep "No services need to be restarted." upgrade.log)
+          ;;
+        *)
+          grep "No services need to be restarted." upgrade.log
+          ;;
+      esac
+    fi
     ;;
   v5)
     curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5.sh | sh