From 9b140b5f5e653deba604291867efa693b4673fd8 Mon Sep 17 00:00:00 2001
From: maimux2x <m0smile10often@gmail.com>
Date: Thu, 6 Feb 2025 12:03:46 +0900
Subject: [PATCH 1/7] Intoroducing Kamal

---
 api/.kamal/hooks/docker-setup.sample      |   3 +
 api/.kamal/hooks/post-app-boot.sample     |   3 +
 api/.kamal/hooks/post-deploy.sample       |  14 +++
 api/.kamal/hooks/post-proxy-reboot.sample |   3 +
 api/.kamal/hooks/pre-app-boot.sample      |   3 +
 api/.kamal/hooks/pre-build.sample         |  51 ++++++++++
 api/.kamal/hooks/pre-connect.sample       |  47 ++++++++++
 api/.kamal/hooks/pre-deploy.sample        | 109 ++++++++++++++++++++++
 api/.kamal/hooks/pre-proxy-reboot.sample  |   3 +
 api/.kamal/secrets                        |  17 ++++
 api/Gemfile                               |   1 +
 api/Gemfile.lock                          |  28 ++++++
 api/config/deploy.staging.yml             |  39 ++++++++
 api/config/deploy.yml                     | 105 +++++++++++++++++++++
 14 files changed, 426 insertions(+)
 create mode 100755 api/.kamal/hooks/docker-setup.sample
 create mode 100755 api/.kamal/hooks/post-app-boot.sample
 create mode 100755 api/.kamal/hooks/post-deploy.sample
 create mode 100755 api/.kamal/hooks/post-proxy-reboot.sample
 create mode 100755 api/.kamal/hooks/pre-app-boot.sample
 create mode 100755 api/.kamal/hooks/pre-build.sample
 create mode 100755 api/.kamal/hooks/pre-connect.sample
 create mode 100755 api/.kamal/hooks/pre-deploy.sample
 create mode 100755 api/.kamal/hooks/pre-proxy-reboot.sample
 create mode 100644 api/.kamal/secrets
 create mode 100644 api/config/deploy.staging.yml
 create mode 100644 api/config/deploy.yml

diff --git a/api/.kamal/hooks/docker-setup.sample b/api/.kamal/hooks/docker-setup.sample
new file mode 100755
index 00000000..2fb07d7d
--- /dev/null
+++ b/api/.kamal/hooks/docker-setup.sample
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Docker set up on $KAMAL_HOSTS..."
diff --git a/api/.kamal/hooks/post-app-boot.sample b/api/.kamal/hooks/post-app-boot.sample
new file mode 100755
index 00000000..70f9c4bc
--- /dev/null
+++ b/api/.kamal/hooks/post-app-boot.sample
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booted app version $KAMAL_VERSION on $KAMAL_HOSTS..."
diff --git a/api/.kamal/hooks/post-deploy.sample b/api/.kamal/hooks/post-deploy.sample
new file mode 100755
index 00000000..75efafc1
--- /dev/null
+++ b/api/.kamal/hooks/post-deploy.sample
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# A sample post-deploy hook
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_ROLE (if set)
+# KAMAL_DESTINATION (if set)
+# KAMAL_RUNTIME
+
+echo "$KAMAL_PERFORMER deployed $KAMAL_VERSION to $KAMAL_DESTINATION in $KAMAL_RUNTIME seconds"
diff --git a/api/.kamal/hooks/post-proxy-reboot.sample b/api/.kamal/hooks/post-proxy-reboot.sample
new file mode 100755
index 00000000..1435a677
--- /dev/null
+++ b/api/.kamal/hooks/post-proxy-reboot.sample
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted kamal-proxy on $KAMAL_HOSTS"
diff --git a/api/.kamal/hooks/pre-app-boot.sample b/api/.kamal/hooks/pre-app-boot.sample
new file mode 100755
index 00000000..45f73550
--- /dev/null
+++ b/api/.kamal/hooks/pre-app-boot.sample
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booting app version $KAMAL_VERSION on $KAMAL_HOSTS..."
diff --git a/api/.kamal/hooks/pre-build.sample b/api/.kamal/hooks/pre-build.sample
new file mode 100755
index 00000000..f87d8113
--- /dev/null
+++ b/api/.kamal/hooks/pre-build.sample
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# A sample pre-build hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_ROLE (if set)
+# KAMAL_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..." >&2
+  git status --porcelain >&2
+  exit 1
+fi
+
+first_remote=$(git remote)
+
+if [ -z "$first_remote" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+current_branch=$(git branch --show-current)
+
+if [ -z "$current_branch" ]; then
+  echo "Not on a git branch, aborting..." >&2
+  exit 1
+fi
+
+remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1)
+
+if [ -z "$remote_head" ]; then
+  echo "Branch not pushed to remote, aborting..." >&2
+  exit 1
+fi
+
+if [ "$KAMAL_VERSION" != "$remote_head" ]; then
+  echo "Version ($KAMAL_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2
+  exit 1
+fi
+
+exit 0
diff --git a/api/.kamal/hooks/pre-connect.sample b/api/.kamal/hooks/pre-connect.sample
new file mode 100755
index 00000000..18e61d7e
--- /dev/null
+++ b/api/.kamal/hooks/pre-connect.sample
@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# A sample pre-connect check
+#
+# Warms DNS before connecting to hosts in parallel
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_ROLE (if set)
+# KAMAL_DESTINATION (if set)
+# KAMAL_RUNTIME
+
+hosts = ENV["KAMAL_HOSTS"].split(",")
+results = nil
+max = 3
+
+elapsed = Benchmark.realtime do
+  results = hosts.map do |host|
+    Thread.new do
+      tries = 1
+
+      begin
+        Socket.getaddrinfo(host, 0, Socket::AF_UNSPEC, Socket::SOCK_STREAM, nil, Socket::AI_CANONNAME)
+      rescue SocketError
+        if tries < max
+          puts "Retrying DNS warmup: #{host}"
+          tries += 1
+          sleep rand
+          retry
+        else
+          puts "DNS warmup failed: #{host}"
+          host
+        end
+      end
+
+      tries
+    end
+  end.map(&:value)
+end
+
+retries = results.sum - hosts.size
+nopes = results.count { |r| r == max }
+
+puts "Prewarmed %d DNS lookups in %.2f sec: %d retries, %d failures" % [ hosts.size, elapsed, retries, nopes ]
diff --git a/api/.kamal/hooks/pre-deploy.sample b/api/.kamal/hooks/pre-deploy.sample
new file mode 100755
index 00000000..1b280c71
--- /dev/null
+++ b/api/.kamal/hooks/pre-deploy.sample
@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+
+# A sample pre-deploy hook
+#
+# Checks the Github status of the build, waiting for a pending build to complete for up to 720 seconds.
+#
+# Fails unless the combined status is "success"
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_COMMAND
+# KAMAL_SUBCOMMAND
+# KAMAL_ROLE (if set)
+# KAMAL_DESTINATION (if set)
+
+# Only check the build status for production deployments
+if ENV["KAMAL_COMMAND"] == "rollback" || ENV["KAMAL_DESTINATION"] != "production"
+  exit 0
+end
+
+require "bundler/inline"
+
+# true = install gems so this is fast on repeat invocations
+gemfile(true, quiet: true) do
+  source "https://rubygems.org"
+
+  gem "octokit"
+  gem "faraday-retry"
+end
+
+MAX_ATTEMPTS = 72
+ATTEMPTS_GAP = 10
+
+def exit_with_error(message)
+  $stderr.puts message
+  exit 1
+end
+
+class GithubStatusChecks
+  attr_reader :remote_url, :git_sha, :github_client, :combined_status
+
+  def initialize
+    @remote_url = `git config --get remote.origin.url`.strip.delete_prefix("https://github.com/")
+    @git_sha = `git rev-parse HEAD`.strip
+    @github_client = Octokit::Client.new(access_token: ENV["GITHUB_TOKEN"])
+    refresh!
+  end
+
+  def refresh!
+    @combined_status = github_client.combined_status(remote_url, git_sha)
+  end
+
+  def state
+    combined_status[:state]
+  end
+
+  def first_status_url
+    first_status = combined_status[:statuses].find { |status| status[:state] == state }
+    first_status && first_status[:target_url]
+  end
+
+  def complete_count
+    combined_status[:statuses].count { |status| status[:state] != "pending"}
+  end
+
+  def total_count
+    combined_status[:statuses].count
+  end
+
+  def current_status
+    if total_count > 0
+      "Completed #{complete_count}/#{total_count} checks, see #{first_status_url} ..."
+    else
+      "Build not started..."
+    end
+  end
+end
+
+
+$stdout.sync = true
+
+puts "Checking build status..."
+attempts = 0
+checks = GithubStatusChecks.new
+
+begin
+  loop do
+    case checks.state
+    when "success"
+      puts "Checks passed, see #{checks.first_status_url}"
+      exit 0
+    when "failure"
+      exit_with_error "Checks failed, see #{checks.first_status_url}"
+    when "pending"
+      attempts += 1
+    end
+
+    exit_with_error "Checks are still pending, gave up after #{MAX_ATTEMPTS * ATTEMPTS_GAP} seconds" if attempts == MAX_ATTEMPTS
+
+    puts checks.current_status
+    sleep(ATTEMPTS_GAP)
+    checks.refresh!
+  end
+rescue Octokit::NotFound
+  exit_with_error "Build status could not be found"
+end
diff --git a/api/.kamal/hooks/pre-proxy-reboot.sample b/api/.kamal/hooks/pre-proxy-reboot.sample
new file mode 100755
index 00000000..061f8059
--- /dev/null
+++ b/api/.kamal/hooks/pre-proxy-reboot.sample
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting kamal-proxy on $KAMAL_HOSTS..."
diff --git a/api/.kamal/secrets b/api/.kamal/secrets
new file mode 100644
index 00000000..b1366604
--- /dev/null
+++ b/api/.kamal/secrets
@@ -0,0 +1,17 @@
+# Secrets defined here are available for reference under registry/password, env/secret, builder/secrets,
+# and accessories/*/env/secret in config/deploy.yml. All secrets should be pulled from either
+# password manager, ENV, or a file. DO NOT ENTER RAW CREDENTIALS HERE! This file needs to be safe for git.
+
+# Option 1: Read secrets from the environment
+KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
+
+# Option 2: Read secrets via a command
+# RAILS_MASTER_KEY=$(cat config/master.key)
+
+# Option 3: Read secrets via kamal secrets helpers
+# These will handle logging in and fetching the secrets in as few calls as possible
+# There are adapters for 1Password, LastPass + Bitwarden
+#
+# SECRETS=$(kamal secrets fetch --adapter 1password --account my-account --from MyVault/MyItem KAMAL_REGISTRY_PASSWORD RAILS_MASTER_KEY)
+# KAMAL_REGISTRY_PASSWORD=$(kamal secrets extract KAMAL_REGISTRY_PASSWORD $SECRETS)
+# RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS)
diff --git a/api/Gemfile b/api/Gemfile
index dcffbfa0..49e72891 100644
--- a/api/Gemfile
+++ b/api/Gemfile
@@ -8,6 +8,7 @@ gem "bootsnap", require: false
 gem "fetch-api"
 gem "jb"
 gem "json"
+gem "kamal", require: false
 gem "metabobank_tools", github: "ddbj/metabobank_tools"
 gem "noodles_gff", path: "../noodles_gff-rb"
 gem "openid_connect"
diff --git a/api/Gemfile.lock b/api/Gemfile.lock
index 53dedcaf..b2bcdfc9 100644
--- a/api/Gemfile.lock
+++ b/api/Gemfile.lock
@@ -121,6 +121,7 @@ GEM
       aws-eventstream (~> 1, >= 1.0.2)
     base62-rb (0.3.1)
     base64 (0.2.0)
+    bcrypt_pbkdf (1.1.1)
     benchmark (0.4.0)
     bigdecimal (3.1.9)
     bindata (2.5.0)
@@ -142,7 +143,9 @@ GEM
       irb (~> 1.10)
       reline (>= 0.3.8)
     diff-lcs (1.5.1)
+    dotenv (3.1.7)
     drb (2.2.1)
+    ed25519 (1.3.0)
     email_validator (2.2.4)
       activemodel
     erubi (1.13.1)
@@ -198,6 +201,17 @@ GEM
       regexp_parser (~> 2.0)
       uri-idna (~> 0.2)
       zeitwerk (~> 2.6)
+    kamal (2.5.2)
+      activesupport (>= 7.0)
+      base64 (~> 0.2)
+      bcrypt_pbkdf (~> 1.0)
+      concurrent-ruby (~> 1.2)
+      dotenv (~> 3.1)
+      ed25519 (~> 1.2)
+      net-ssh (~> 7.3)
+      sshkit (>= 1.23.0, < 2.0)
+      thor (~> 1.3)
+      zeitwerk (>= 2.6.18, < 3.0)
     language_server-protocol (3.17.0.4)
     logger (1.6.5)
     loofah (2.24.0)
@@ -221,8 +235,13 @@ GEM
       net-protocol
     net-protocol (0.2.2)
       timeout
+    net-scp (4.1.0)
+      net-ssh (>= 2.6.5, < 8.0.0)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
     net-smtp (0.5.1)
       net-protocol
+    net-ssh (7.3.0)
     nio4r (2.7.4)
     nokogiri (1.18.2-aarch64-linux-gnu)
       racc (~> 1.4)
@@ -244,6 +263,7 @@ GEM
       tzinfo
       validate_url
       webfinger (~> 2.0)
+    ostruct (0.6.1)
     pagy (9.3.3)
     parallel (1.26.3)
     parser (3.3.7.1)
@@ -394,6 +414,13 @@ GEM
       fugit (~> 1.11.0)
       railties (>= 7.1)
       thor (~> 1.3.1)
+    sshkit (1.24.0)
+      base64
+      logger
+      net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
+      net-ssh (>= 2.8.0)
+      ostruct
     stringio (3.1.2)
     swd (2.0.3)
       activesupport (>= 3)
@@ -448,6 +475,7 @@ DEPENDENCIES
   fetch-api
   jb
   json
+  kamal
   metabobank_tools!
   noodles_gff!
   openid_connect
diff --git a/api/config/deploy.staging.yml b/api/config/deploy.staging.yml
new file mode 100644
index 00000000..8ce4e6e5
--- /dev/null
+++ b/api/config/deploy.staging.yml
@@ -0,0 +1,39 @@
+servers:
+  web:
+    - repository-staging 
+
+proxy:
+  host: repository-dev.ddbj.nig.ac.jp
+
+env:
+  clear:
+    API_URL:            https://repository-dev.ddbj.nig.ac.jp/api
+    MINIO_ENDPOINT:     https://repository-storage-dev.ddbj.nig.ac.jp
+    OIDC_ISSUER_URL:    https://accounts-dev.ddbj.nig.ac.jp/realms/master
+    SENTRY_CURRENT_ENV: staging
+    DDBJ_VALIDATOR_URL: http://validator-staging:3000/api
+
+volumes:
+  - "repository_storage:/app/api/storage"
+
+accessories:
+  postgres:
+    host: repository-staging
+
+    volumes:
+      - ./volumes/repository-staging/postgres:/var/lib/postgresql/data
+
+  minio:
+    host: repository-staging
+
+    volumes:
+      - ./volumes/repository-staging/minio:/data
+
+  virtuoso:
+    host: repository-staging
+
+    volumes:
+      - ./volumes/repository-staging/virtuoso:/database
+
+  validator:
+    host: repository-staging
diff --git a/api/config/deploy.yml b/api/config/deploy.yml
new file mode 100644
index 00000000..71a9ae94
--- /dev/null
+++ b/api/config/deploy.yml
@@ -0,0 +1,105 @@
+service: repository 
+image: w3const/repository
+require_destination: true
+
+registry:
+  username: w3const
+
+  password:
+    - KAMAL_REGISTRY_PASSWORD
+
+env:
+  secret:
+    - DATABASE_URL
+    - DATABASE_URL_DRASEARCH
+    - DATABASE_URL_DWAY
+    - RAILS_MASTER_KEY
+    - MINIO_SECRET_ACCESS_KEY
+    - SMTP_USERNAME
+    - SMTP_PASSWORD
+    - SECRET_KEY_BASE
+    - SENTRY_DSN
+
+  clear:
+    MINIO_ACCESS_KEY_ID: ddbj-repository
+    MINIO_BUCKET:        uploads
+    OIDC_CLIENT_ID:      ddbj-repository
+    REPOSITORY_DIR:      /data/repository
+    SMTP_ADDRESS:        smtp.gmail.com
+    SMTP_AUTHENTICATION: plain
+    SMTP_DOMAIN:         ddbj.nig.ac.jp
+    SMTP_PORT:           587
+    SOLID_QUEUE_IN_PUMA: 'true'
+    TZ:                  Asia/Tokyo
+    USER_HOME_DIR:       /data/home
+
+aliases:
+  console: app exec --interactive --reuse "bin/rails console"
+  shell: app exec --interactive --reuse "bash"
+  logs: app logs -f
+  dbc: app exec --interactive --reuse "bin/rails dbconsole"
+
+asset_path: /app/api/public/assets
+
+builder:
+  arch: amd64
+
+ssh:
+  user: w3const
+
+accessories:
+  postgres:
+    image: postgres:16
+    options:
+      user: 2233:11370
+    env:
+      secret:
+        - POSTGRES_PASSWORD
+      clear:
+        POSTGRES_DB: repository
+        TZ:          Asia/Tokyo
+        
+  minio:
+    image: minio/minio:RELEASE.2023-12-07T04-16-00Z
+    cmd: server /data --console-address :9001
+    options:
+      user: 2233:11370
+    env:
+      secret:
+        - MINIO_ROOT_USER
+        - MINIO_ROOT_PASSWORD
+      clear:
+        TZ: Asia/Tokyo
+  
+  virtuoso:
+    image: openlink/virtuoso-opensource-7:7.2.6-r1-g0a3336c
+    options:
+      user: 2233:11370
+    env:
+      secret:
+        - DBA_PASSWORD
+      clear:
+        VIRT_Client_SQL_PREFETCH_BYTES: '160000'
+        VIRT_Client_SQL_PREFETCH_ROWS: '10000'
+        VIRT_Parameters_DirsAllowed: ., ../vad, /usr/share/proj, /database
+        VIRT_Parameters_MaxDirtyBuffers: '130000'
+        VIRT_Parameters_NumberOfBuffers: '170000'
+        VIRT_SPARQL_MaxQueryCostEstimationTime: '-1'
+        VIRT_SPARQL_MaxQueryExecutionTime: '300'
+        VIRT_SPARQL_MaxSortedTopRows: '100000'
+        VIRT_SPARQL_ResultSetMaxRows: '1000000'
+
+  validator:
+    image: ghcr.io/ddbj/ddbj_validator:main
+    env:
+      secret:
+        - DDBJ_VALIDATOR_APP_POSTGRES_PASSWD
+        - DDBJ_VALIDATOR_APP_POSTGRES_USER
+      clear:
+        DDBJ_VALIDATOR_APP_POSTGRES_HOST:             at098
+        DDBJ_VALIDATOR_APP_POSTGRES_PORT:             54301
+        DDBJ_VALIDATOR_APP_NAMED_GRAPHE_URI_TAXONOMY: http://ddbj.nig.ac.jp/ontologies/taxonomy-private
+        DDBJ_VALIDATOR_APP_POSTGRES_TIMEOUT:          '30'
+        DDBJ_VALIDATOR_APP_VALIDATOR_LOG_DIR:         /usr/src/ddbj_validator/logs
+        DDBJ_VALIDATOR_APP_VIRTUOSO_ENDPOINT_MASTER:  http://virtuoso:8890/sparql
+        TZ:                                           Asia/Tokyo

From 5277751eeaa0f2a9383eda90894888029d87c5d0 Mon Sep 17 00:00:00 2001
From: maimux2x <m0smile10often@gmail.com>
Date: Wed, 12 Feb 2025 12:02:47 +0900
Subject: [PATCH 2/7] Add secrets

---
 api/.kamal/secrets         | 17 -----------------
 api/.kamal/secrets-common  |  8 ++++++++
 api/.kamal/secrets.staging |  7 +++++++
 3 files changed, 15 insertions(+), 17 deletions(-)
 delete mode 100644 api/.kamal/secrets
 create mode 100644 api/.kamal/secrets-common
 create mode 100644 api/.kamal/secrets.staging

diff --git a/api/.kamal/secrets b/api/.kamal/secrets
deleted file mode 100644
index b1366604..00000000
--- a/api/.kamal/secrets
+++ /dev/null
@@ -1,17 +0,0 @@
-# Secrets defined here are available for reference under registry/password, env/secret, builder/secrets,
-# and accessories/*/env/secret in config/deploy.yml. All secrets should be pulled from either
-# password manager, ENV, or a file. DO NOT ENTER RAW CREDENTIALS HERE! This file needs to be safe for git.
-
-# Option 1: Read secrets from the environment
-KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
-
-# Option 2: Read secrets via a command
-# RAILS_MASTER_KEY=$(cat config/master.key)
-
-# Option 3: Read secrets via kamal secrets helpers
-# These will handle logging in and fetching the secrets in as few calls as possible
-# There are adapters for 1Password, LastPass + Bitwarden
-#
-# SECRETS=$(kamal secrets fetch --adapter 1password --account my-account --from MyVault/MyItem KAMAL_REGISTRY_PASSWORD RAILS_MASTER_KEY)
-# KAMAL_REGISTRY_PASSWORD=$(kamal secrets extract KAMAL_REGISTRY_PASSWORD $SECRETS)
-# RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS)
diff --git a/api/.kamal/secrets-common b/api/.kamal/secrets-common
new file mode 100644
index 00000000..f4466983
--- /dev/null
+++ b/api/.kamal/secrets-common
@@ -0,0 +1,8 @@
+DATABASE_URL_DRASEARCH=$COMMON_DATABASE_URL_DRASEARCH
+DATABASE_URL_DWAY=$COMMON_DATABASE_URL_DWAY
+DDBJ_VALIDATOR_APP_POSTGRES_PASSWD=$COMMON_DDBJ_VALIDATOR_APP_POSTGRES_PASSWD
+DDBJ_VALIDATOR_APP_POSTGRES_USER=$COMMON_DDBJ_VALIDATOR_APP_POSTGRES_USER
+KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
+SENTRY_DSN=$COMMON_SENTRY_DSN
+SMTP_PASSWORD=$COMMON_SMTP_PASSWORD
+SMTP_USERNAME=$COMMON_SMTP_USERNAME
diff --git a/api/.kamal/secrets.staging b/api/.kamal/secrets.staging
new file mode 100644
index 00000000..e6559500
--- /dev/null
+++ b/api/.kamal/secrets.staging
@@ -0,0 +1,7 @@
+DATABASE_URL=$STAGING_DATABASE_URL
+DBA_PASSWORD=$STAGING_DBA_PASSWORD
+MINIO_ROOT_PASSWORD=$STAGING_MINIO_ROOT_PASSWORD
+MINIO_ROOT_USER=$STAGING_MINIO_ROOT_USER
+MINIO_SECRET_ACCESS_KEY=$STAGING_MINIO_SECRET_ACCESS_KEY
+POSTGRES_PASSWORD=$STAGING_POSTGRES_PASSWORD
+SECRET_KEY_BASE=$STAGING_SECRET_KEY_BASE

From 9e75f3dc86e2b29bf086c949c41ee1db3b655d8d Mon Sep 17 00:00:00 2001
From: maimux2x <m0smile10often@gmail.com>
Date: Thu, 13 Feb 2025 09:50:02 +0900
Subject: [PATCH 3/7] Remove unused files

---
 ansible.cfg                 |  2 -
 ansible/deploy.yml          | 15 --------
 ansible/healthcheck         | 50 ------------------------
 ansible/healthcheck.yml     | 23 -----------
 ansible/hosts               |  5 ---
 ansible/setup.yml           | 25 ------------
 compose/.gitignore          |  2 -
 compose/_exec               | 17 --------
 compose/base.yml            | 77 -------------------------------------
 compose/local               |  1 -
 compose/local.env           | 20 ----------
 compose/local.yml           | 41 --------------------
 compose/production          |  1 -
 compose/production.yml      | 19 ---------
 compose/staging             |  1 -
 compose/staging.yml         |  1 -
 compose/varnish/Dockerfile  |  3 --
 compose/varnish/default.vcl | 27 -------------
 18 files changed, 330 deletions(-)
 delete mode 100644 ansible.cfg
 delete mode 100644 ansible/deploy.yml
 delete mode 100755 ansible/healthcheck
 delete mode 100644 ansible/healthcheck.yml
 delete mode 100644 ansible/hosts
 delete mode 100644 ansible/setup.yml
 delete mode 100644 compose/.gitignore
 delete mode 100755 compose/_exec
 delete mode 100644 compose/base.yml
 delete mode 120000 compose/local
 delete mode 100644 compose/local.env
 delete mode 100644 compose/local.yml
 delete mode 120000 compose/production
 delete mode 100644 compose/production.yml
 delete mode 120000 compose/staging
 delete mode 120000 compose/staging.yml
 delete mode 100644 compose/varnish/Dockerfile
 delete mode 100644 compose/varnish/default.vcl

diff --git a/ansible.cfg b/ansible.cfg
deleted file mode 100644
index 7eaac5a2..00000000
--- a/ansible.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-[defaults]
-inventory = ansible/hosts
diff --git a/ansible/deploy.yml b/ansible/deploy.yml
deleted file mode 100644
index 11ed0e65..00000000
--- a/ansible/deploy.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-- hosts: at026
-
-  vars_prompt:
-    - name: stage
-      prompt: stage
-      private: no
-
-  tasks:
-    - git:
-        repo: https://github.com/ddbj/ddbj-repository.git
-        dest: ~/ddbj-repository
-
-    - command: compose/{{ stage }} up --build --detach
-      args:
-        chdir: ~/ddbj-repository
diff --git a/ansible/healthcheck b/ansible/healthcheck
deleted file mode 100755
index 12079800..00000000
--- a/ansible/healthcheck
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'erb'
-require 'json'
-require 'open3'
-require 'tempfile'
-
-def send_email(subject: nil, body: nil)
-  recipients = File.readlines("#{ENV['HOME']}/.healthcheck/recipients").map(&:chomp).reject(&:empty?)
-
-  Tempfile.open 'healthcheck-ddbj-repository' do |f|
-    f.puts body
-    f.flush
-
-    system 'sendgmail_w3const.py', '--sj', subject, '--to', recipients.join(','), '--body', f.path
-  end
-end
-
-ENV['PATH'] += ":/cm/local/apps/docker/current/bin:#{ENV['HOME']}/w3const_base/curatortool"
-
-stage = ARGV[0]
-out, status = Open3.capture2e("compose/#{stage} ps --all --format json")
-
-unless status.success?
-  send_email subject: "[ddbj-repository][#{stage}] Healthcheck failed", body: <<-ERROR
-Error:
-#{out}
-  ERROR
-
-  exit 1
-end
-
-failed_services = out.lines.map {|line| JSON.parse(line) }.select {|service|
-  state, exit_code = service.values_at('State', 'ExitCode')
-
-  state == 'exited' && !(exit_code == 0 || exit_code == 143)
-}
-
-exit 0 if failed_services.empty?
-
-send_email subject: "[ddbj-repository][#{stage}] Healthcheck failed", body: ERB.new(<<-ERB, nil, '-').result
-Summary:
-Several services of docker compose have been failed.
-Please check the error messages and restart them.
-
-Failed services:
-<%- failed_services.each do |service| -%>
-- <%= service['Service'] %>
-<%- end -%>
-ERB
diff --git a/ansible/healthcheck.yml b/ansible/healthcheck.yml
deleted file mode 100644
index 496d2be1..00000000
--- a/ansible/healthcheck.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-- hosts: all
-
-  tasks:
-    - file:
-        path: /home/w3const/bin
-        state: directory
-
-    - get_url:
-        url: https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64
-        dest: /home/w3const/bin/jq
-        mode: '0755'
-
-    - git:
-        repo: https://github.com/ddbj/w3const_base.git
-        dest: ~/w3const_base
-
-    - cron:
-        name: healthcheck-ddbj-repository-{{ item }}
-        minute: '*/10'
-        job: 'cd /home/w3const/ddbj-repository/deploy && ansible/healthcheck {{ item }} > /dev/null'
-      loop:
-        - staging
-        - production
diff --git a/ansible/hosts b/ansible/hosts
deleted file mode 100644
index afb5c32e..00000000
--- a/ansible/hosts
+++ /dev/null
@@ -1,5 +0,0 @@
-at025
-at026
-
-[all:vars]
-ansible_user=w3const
diff --git a/ansible/setup.yml b/ansible/setup.yml
deleted file mode 100644
index 5edf60b4..00000000
--- a/ansible/setup.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-- hosts: all
-  tasks:
-    - file:
-        path: ~/.docker/cli-plugins
-        state: directory
-
-    - get_url:
-        url: https://github.com/docker/compose/releases/download/v2.24.5/docker-compose-linux-x86_64
-        dest: ~/.docker/cli-plugins/docker-compose
-        mode: +x
-
-    - git:
-        repo: https://github.com/ddbj/ddbj-repository.git
-        dest: ~/ddbj-repository/deploy
-
-    - file:
-        path: "{{ item.1 | regex_replace('STAGE', item.0) }}"
-        state: directory
-      loop: '{{ stages | product(paths) }}'
-      vars:
-        stages:
-          - staging
-          - production
-        paths:
-          - ~/repository/STAGE
diff --git a/compose/.gitignore b/compose/.gitignore
deleted file mode 100644
index 56135b9b..00000000
--- a/compose/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/*.env
-!/local.env
diff --git a/compose/_exec b/compose/_exec
deleted file mode 100755
index 694f9e10..00000000
--- a/compose/_exec
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-set -e
-
-export STAGE=${0##*/}
-
-dir=${0%/*}
-
-export COMPOSE_FILE=$dir/base.yml:$dir/$STAGE.yml
-export COMPOSE_PROJECT_NAME=repository-$STAGE
-export DOCKER_BUILDKIT=1
-
-set -o allexport
-source $dir/$STAGE.env
-set +o allexport
-
-exec docker compose "$@"
diff --git a/compose/base.yml b/compose/base.yml
deleted file mode 100644
index 732153fe..00000000
--- a/compose/base.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-services:
-  varnish:
-    build: varnish
-    user: root # https://github.com/varnish/docker-varnish/issues/53
-
-    environment:
-      VARNISH_HTTP_PORT: 8080
-
-    ports:
-      - ${VARNISH_PORT:?}:8080
-
-    tmpfs:
-      - /var/lib/varnish/varnishd:exec
-
-    depends_on:
-      rails:
-        condition: service_started
-
-  rails:
-    build:
-      context: ../
-      dockerfile: api/Dockerfile
-
-      args:
-        API_URL:
-        APP_GID:
-        APP_UID:
-        NODE_VERSION:
-        RUBY_VERSION:
-
-    init: true
-
-    environment:
-      API_URL:
-      DATABASE_URL:
-      DATABASE_URL_DRASEARCH:
-      DATABASE_URL_DWAY:
-      DDBJ_VALIDATOR_URL:      http://ddbj-validator:3000/api
-      DISABLE_SSL:
-      MINIO_ACCESS_KEY_ID:
-      MINIO_BUCKET:            uploads
-      MINIO_ENDPOINT:
-      MINIO_SECRET_ACCESS_KEY:
-      OIDC_CLIENT_ID:
-      OIDC_ISSUER_URL:
-      RAILS_MAX_THREADS:       16
-      REPOSITORY_DIR:          /data/repository
-      SECRET_KEY_BASE:
-      SENTRY_CURRENT_ENV:
-      SENTRY_DSN:
-      SMTP_ADDRESS:
-      SMTP_AUTHENTICATION:
-      SMTP_DOMAIN:
-      SMTP_PASSWORD:
-      SMTP_PORT:
-      SMTP_USERNAME:
-      SOLID_QUEUE_IN_PUMA:     'true'
-      TZ:                      Japan
-      USER_HOME_DIR:           /data/home
-
-    depends_on:
-      - ddbj-validator
-
-  ddbj-validator:
-    build:
-      context: ../ddbj_validator
-
-    environment:
-      DDBJ_VALIDATOR_APP_NAMED_GRAPHE_URI_TAXONOMY: http://ddbj.nig.ac.jp/ontologies/taxonomy-private
-      DDBJ_VALIDATOR_APP_POSTGRES_HOST:
-      DDBJ_VALIDATOR_APP_POSTGRES_PASSWD:
-      DDBJ_VALIDATOR_APP_POSTGRES_PORT:
-      DDBJ_VALIDATOR_APP_POSTGRES_TIMEOUT:          '30'
-      DDBJ_VALIDATOR_APP_POSTGRES_USER:
-      DDBJ_VALIDATOR_APP_VALIDATOR_LOG_DIR:         /usr/src/ddbj_validator/logs
-      DDBJ_VALIDATOR_APP_VIRTUOSO_ENDPOINT_MASTER:  http://virtuoso:8890/sparql
-      TZ:                                           Japan
diff --git a/compose/local b/compose/local
deleted file mode 120000
index 87d6f3fa..00000000
--- a/compose/local
+++ /dev/null
@@ -1 +0,0 @@
-_exec
\ No newline at end of file
diff --git a/compose/local.env b/compose/local.env
deleted file mode 100644
index dadc46d9..00000000
--- a/compose/local.env
+++ /dev/null
@@ -1,20 +0,0 @@
-API_URL=http://localhost:3000/api
-APP_GID=${APP_GID:?}
-APP_UID=${APP_UID:?}
-DATABASE_URL=postgres://postgres@postgres
-DATABASE_URL_DRASEARCH=postgres://postgres@postgres
-DATABASE_URL_DWAY=postgres://postgres@postgres
-DISABLE_SSL=true
-MINIO_ACCESS_KEY_ID=repository
-MINIO_ENDPOINT=http://repository.localhost:3000
-MINIO_SECRET_ACCESS_KEY=changeme
-OIDC_CLIENT_ID=ddbj-repository
-OIDC_ISSUER_URL=https://accounts-staging.ddbj.nig.ac.jp/realms/master
-SECRET_KEY_BASE=changeme
-SMTP_ADDRESS=
-SMTP_AUTHENTICATION=
-SMTP_DOMAIN=
-SMTP_PASSWORD=
-SMTP_PORT=
-SMTP_USERNAME=
-VARNISH_PORT=3000
diff --git a/compose/local.yml b/compose/local.yml
deleted file mode 100644
index bb05be1d..00000000
--- a/compose/local.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-volumes:
-  postgres:
-  minio:
-
-services:
-  rails:
-    extra_hosts:
-      - repository.localhost:host-gateway
-
-    volumes:
-      - ../api/tmp/home:/data/home
-      - ../api/tmp/repository:/data/repository
-
-    depends_on:
-      postgres:
-        condition: service_healthy
-      minio:
-        condition: service_started
-
-  postgres:
-    image: postgres:16
-
-    environment:
-      POSTGRES_DB: repository
-      POSTGRES_HOST_AUTH_METHOD: trust
-
-    volumes:
-      - postgres:/var/lib/postgresql/data
-
-    healthcheck:
-      test: [CMD-SHELL, pg_isready]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  minio:
-    image: minio/minio
-    command: [server, /data]
-
-    volumes:
-      - minio:/data
diff --git a/compose/production b/compose/production
deleted file mode 120000
index 87d6f3fa..00000000
--- a/compose/production
+++ /dev/null
@@ -1 +0,0 @@
-_exec
\ No newline at end of file
diff --git a/compose/production.yml b/compose/production.yml
deleted file mode 100644
index 1683cc65..00000000
--- a/compose/production.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-services:
-  rails:
-    group_add:
-      - '60000' # submitter
-
-    volumes:
-      - type: bind
-        source: ~/repository/$STAGE
-        target: /data/repository
-      - type: bind
-        source: /home
-        target: /data/home
-
-    restart: unless-stopped
-
-networks:
-  default:
-    external: true
-    name: repository-$STAGE
diff --git a/compose/staging b/compose/staging
deleted file mode 120000
index 87d6f3fa..00000000
--- a/compose/staging
+++ /dev/null
@@ -1 +0,0 @@
-_exec
\ No newline at end of file
diff --git a/compose/staging.yml b/compose/staging.yml
deleted file mode 120000
index 8dfd7d36..00000000
--- a/compose/staging.yml
+++ /dev/null
@@ -1 +0,0 @@
-production.yml
\ No newline at end of file
diff --git a/compose/varnish/Dockerfile b/compose/varnish/Dockerfile
deleted file mode 100644
index f3944492..00000000
--- a/compose/varnish/Dockerfile
+++ /dev/null
@@ -1,3 +0,0 @@
-FROM varnish:7.6.1
-
-COPY default.vcl /etc/varnish/
diff --git a/compose/varnish/default.vcl b/compose/varnish/default.vcl
deleted file mode 100644
index 37705d97..00000000
--- a/compose/varnish/default.vcl
+++ /dev/null
@@ -1,27 +0,0 @@
-vcl 4.1;
-
-backend rails {
-  .host = "rails:80";
-}
-
-backend minio {
-  .host = "minio:9000";
-}
-
-sub vcl_recv {
-  if (req.url ~ "^/uploads/") {
-    set req.backend_hint = minio;
-
-    if (req.url ~ "X-Amz-Signature=") {
-      unset req.http.Authorization;
-    }
-
-    return (pipe);
-  } else {
-    set req.backend_hint = rails;
-  }
-}
-
-sub vcl_hash {
-  hash_data(req.http.X-Forwarded-Proto);
-}

From 05ee74b230f379fc036f2179019e0f5a229064ae Mon Sep 17 00:00:00 2001
From: maimux2x <m0smile10often@gmail.com>
Date: Thu, 13 Feb 2025 10:23:48 +0900
Subject: [PATCH 4/7] Add args for builder

---
 api/config/deploy.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/api/config/deploy.yml b/api/config/deploy.yml
index 71a9ae94..54556509 100644
--- a/api/config/deploy.yml
+++ b/api/config/deploy.yml
@@ -43,6 +43,9 @@ asset_path: /app/api/public/assets
 
 builder:
   arch: amd64
+  args:
+    APP_GID: 11370
+    APP_UID: 2233
 
 ssh:
   user: w3const

From 2a822edb6e3c3e962ab8d5460c45ef4eddfae15e Mon Sep 17 00:00:00 2001
From: maimux2x <m0smile10often@gmail.com>
Date: Thu, 13 Feb 2025 11:09:14 +0900
Subject: [PATCH 5/7] Fix volumes

---
 api/config/deploy.staging.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/config/deploy.staging.yml b/api/config/deploy.staging.yml
index 8ce4e6e5..f34f1b77 100644
--- a/api/config/deploy.staging.yml
+++ b/api/config/deploy.staging.yml
@@ -14,7 +14,7 @@ env:
     DDBJ_VALIDATOR_URL: http://validator-staging:3000/api
 
 volumes:
-  - "repository_storage:/app/api/storage"
+  - ./volumes/repository-staging/storage:/app/api/storage
 
 accessories:
   postgres:

From a4a3e865fe9a6afcdca663e707569f850ff18bbb Mon Sep 17 00:00:00 2001
From: maimux2x <m0smile10often@gmail.com>
Date: Thu, 13 Feb 2025 11:25:45 +0900
Subject: [PATCH 6/7] Fix minio settings

---
 api/.kamal/secrets-common | 1 +
 api/config/deploy.yml     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/api/.kamal/secrets-common b/api/.kamal/secrets-common
index f4466983..c8e36509 100644
--- a/api/.kamal/secrets-common
+++ b/api/.kamal/secrets-common
@@ -3,6 +3,7 @@ DATABASE_URL_DWAY=$COMMON_DATABASE_URL_DWAY
 DDBJ_VALIDATOR_APP_POSTGRES_PASSWD=$COMMON_DDBJ_VALIDATOR_APP_POSTGRES_PASSWD
 DDBJ_VALIDATOR_APP_POSTGRES_USER=$COMMON_DDBJ_VALIDATOR_APP_POSTGRES_USER
 KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
+MINIO_ACCESS_KEY_ID=$COMMON_MINIO_ACCESS_KEY_ID
 SENTRY_DSN=$COMMON_SENTRY_DSN
 SMTP_PASSWORD=$COMMON_SMTP_PASSWORD
 SMTP_USERNAME=$COMMON_SMTP_USERNAME
diff --git a/api/config/deploy.yml b/api/config/deploy.yml
index 54556509..45b3a8ea 100644
--- a/api/config/deploy.yml
+++ b/api/config/deploy.yml
@@ -14,6 +14,7 @@ env:
     - DATABASE_URL_DRASEARCH
     - DATABASE_URL_DWAY
     - RAILS_MASTER_KEY
+    - MINIO_ACCESS_KEY_ID
     - MINIO_SECRET_ACCESS_KEY
     - SMTP_USERNAME
     - SMTP_PASSWORD
@@ -21,7 +22,6 @@ env:
     - SENTRY_DSN
 
   clear:
-    MINIO_ACCESS_KEY_ID: ddbj-repository
     MINIO_BUCKET:        uploads
     OIDC_CLIENT_ID:      ddbj-repository
     REPOSITORY_DIR:      /data/repository

From a841ea5593c931885b2706ecd422be988318f0df Mon Sep 17 00:00:00 2001
From: maimux2x <m0smile10often@gmail.com>
Date: Thu, 13 Feb 2025 12:03:57 +0900
Subject: [PATCH 7/7] Add missing settings to deploy.yml

---
 api/config/deploy.staging.yml | 10 +++++++---
 api/config/deploy.yml         |  1 +
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/api/config/deploy.staging.yml b/api/config/deploy.staging.yml
index f34f1b77..28860d07 100644
--- a/api/config/deploy.staging.yml
+++ b/api/config/deploy.staging.yml
@@ -1,20 +1,24 @@
 servers:
   web:
-    - repository-staging 
+    hosts:
+      - repository-staging 
+    options:
+      group_add: 60000
 
 proxy:
   host: repository-dev.ddbj.nig.ac.jp
 
 env:
   clear:
-    API_URL:            https://repository-dev.ddbj.nig.ac.jp/api
     MINIO_ENDPOINT:     https://repository-storage-dev.ddbj.nig.ac.jp
     OIDC_ISSUER_URL:    https://accounts-dev.ddbj.nig.ac.jp/realms/master
     SENTRY_CURRENT_ENV: staging
-    DDBJ_VALIDATOR_URL: http://validator-staging:3000/api
+    DDBJ_VALIDATOR_URL: http://repository-validator:3000/api
 
 volumes:
   - ./volumes/repository-staging/storage:/app/api/storage
+  - ~/repository/staging:/data/repository
+  - /home:/data/home
 
 accessories:
   postgres:
diff --git a/api/config/deploy.yml b/api/config/deploy.yml
index 45b3a8ea..2fb3c834 100644
--- a/api/config/deploy.yml
+++ b/api/config/deploy.yml
@@ -46,6 +46,7 @@ builder:
   args:
     APP_GID: 11370
     APP_UID: 2233
+    API_URL: https://repository-dev.ddbj.nig.ac.jp/api
 
 ssh:
   user: w3const