From af7309371acfaf73cfdcff3c18e39cce138b6544 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 9 Dec 2024 08:13:29 +0000 Subject: [PATCH 1/4] envsubst: remove explicit subst of exported vars For nginx config, a new approach is implemented with mawk. This is similar to envusbst, but more ergonomic: * no need to explicitly list all variables * throw error on missing variables * do not replace nginx vars like $host, $request_uri with empty strings (in contrast to envsubst when executed without an explicit variable list) Risks: There are a couple of changes which may break existing deployments: * changing client-config.json.template * requiring all substituted variable to be defined Closes #473 --- .../workflows/{test-nginx.yml => test.yml} | 18 +++++-- enketo.dockerfile | 1 + files/enketo/start-enketo.sh | 2 +- files/nginx/client-config.json.template | 2 +- files/nginx/setup-odk.sh | 7 +-- files/service/scripts/start-odk.sh | 2 +- files/shared/envsub.awk | 39 +++++++++++++++ nginx.dockerfile | 4 +- service.dockerfile | 1 + test/envsub/bad-example.in | 3 ++ test/envsub/bad-example.stderr.expected | 5 ++ test/envsub/bad-example.stdout.expected | 3 ++ test/envsub/good-example.expected | 20 ++++++++ test/envsub/good-example.in | 20 ++++++++ test/envsub/run-tests.sh | 50 +++++++++++++++++++ test/{ => nginx}/.gitignore | 0 .../files/nginx-test/acme-challenge | 0 .../files/nginx-test/http_root/index.html | 0 .../nginx-test/http_root/should-be-cached.txt | 0 .../files/nginx-test/http_root/version.txt | 0 test/{ => nginx}/mock-http-server/.gitignore | 0 test/{ => nginx}/mock-http-server/index.js | 0 .../mock-http-server/package-lock.json | 0 .../{ => nginx}/mock-http-server/package.json | 0 test/{ => nginx}/mock-http-service.dockerfile | 0 .../{ => nginx}/nginx.test.docker-compose.yml | 11 ++-- test/{ => nginx}/package-lock.json | 0 test/{ => nginx}/package.json | 0 test/{ => nginx}/run-tests.sh | 0 test/{ => nginx}/test-nginx.js | 0 30 files changed, 173 insertions(+), 15 deletions(-) rename .github/workflows/{test-nginx.yml => test.yml} (58%) create mode 100755 files/shared/envsub.awk create mode 100644 test/envsub/bad-example.in create mode 100644 test/envsub/bad-example.stderr.expected create mode 100644 test/envsub/bad-example.stdout.expected create mode 100644 test/envsub/good-example.expected create mode 100644 test/envsub/good-example.in create mode 100755 test/envsub/run-tests.sh rename test/{ => nginx}/.gitignore (100%) rename test/{ => nginx}/files/nginx-test/acme-challenge (100%) rename test/{ => nginx}/files/nginx-test/http_root/index.html (100%) rename test/{ => nginx}/files/nginx-test/http_root/should-be-cached.txt (100%) rename test/{ => nginx}/files/nginx-test/http_root/version.txt (100%) rename test/{ => nginx}/mock-http-server/.gitignore (100%) rename test/{ => nginx}/mock-http-server/index.js (100%) rename test/{ => nginx}/mock-http-server/package-lock.json (100%) rename test/{ => nginx}/mock-http-server/package.json (100%) rename test/{ => nginx}/mock-http-service.dockerfile (100%) rename test/{ => nginx}/nginx.test.docker-compose.yml (66%) rename test/{ => nginx}/package-lock.json (100%) rename test/{ => nginx}/package.json (100%) rename test/{ => nginx}/run-tests.sh (100%) rename test/{ => nginx}/test-nginx.js (100%) diff --git a/.github/workflows/test-nginx.yml b/.github/workflows/test.yml similarity index 58% rename from .github/workflows/test-nginx.yml rename to .github/workflows/test.yml index ee56a757..944b94bc 100644 --- a/.github/workflows/test-nginx.yml +++ b/.github/workflows/test.yml @@ -1,11 +1,21 @@ -name: Test nginx config +name: Tests on: push: pull_request: jobs: - build: + test-envsubst: + timeout-minutes: 2 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - run: cd test/envsub && ./run-tests.sh + test-nginx: timeout-minutes: 10 runs-on: ubuntu-latest steps: @@ -17,8 +27,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 20.17.0 - - run: cd test && npm i - - run: cd test && ./run-tests.sh + - run: cd test/nginx && npm i + - run: cd test/nginx && ./run-tests.sh - if: always() run: docker logs test-nginx-1 || true diff --git a/enketo.dockerfile b/enketo.dockerfile index c4b03466..a9fb58f6 100644 --- a/enketo.dockerfile +++ b/enketo.dockerfile @@ -10,6 +10,7 @@ WORKDIR ${ENKETO_SRC_DIR} # care about anything the server needs. because the client config is baked at # build time, we therefore hand it the untemplated config. +COPY files/shared/envsub.awk /scripts/ COPY files/enketo/config.json.template ${ENKETO_SRC_DIR}/config/config.json.template COPY files/enketo/config.json.template ${ENKETO_SRC_DIR}/config/config.json COPY files/enketo/start-enketo.sh ${ENKETO_SRC_DIR}/start-enketo.sh diff --git a/files/enketo/start-enketo.sh b/files/enketo/start-enketo.sh index b9a43ac0..b67aa5cd 100755 --- a/files/enketo/start-enketo.sh +++ b/files/enketo/start-enketo.sh @@ -7,7 +7,7 @@ BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https: SECRET=$(cat /etc/secrets/enketo-secret) \ LESS_SECRET=$(cat /etc/secrets/enketo-less-secret) \ API_KEY=$(cat /etc/secrets/enketo-api-key) \ -envsubst '$DOMAIN $BASE_URL $SECRET $LESS_SECRET $API_KEY $SUPPORT_EMAIL' \ +/scripts/envsub.awk \ < "$CONFIG_PATH.template" \ > "$CONFIG_PATH" diff --git a/files/nginx/client-config.json.template b/files/nginx/client-config.json.template index 3387eef3..f1965a49 100644 --- a/files/nginx/client-config.json.template +++ b/files/nginx/client-config.json.template @@ -1,3 +1,3 @@ { - "oidcEnabled": $OIDC_ENABLED + "oidcEnabled": ${OIDC_ENABLED} } diff --git a/files/nginx/setup-odk.sh b/files/nginx/setup-odk.sh index db0f9356..1f33b776 100644 --- a/files/nginx/setup-odk.sh +++ b/files/nginx/setup-odk.sh @@ -1,13 +1,14 @@ #!/bin/bash - echo "writing client config..." if [[ $OIDC_ENABLED != 'true' ]] && [[ $OIDC_ENABLED != 'false' ]]; then echo 'OIDC_ENABLED must be either true or false' exit 1 fi -envsubst < /usr/share/odk/nginx/client-config.json.template > /usr/share/nginx/html/client-config.json +/scripts/envsub.awk \ + < /usr/share/odk/nginx/client-config.json.template \ + > /usr/share/nginx/html/client-config.json DH_PATH=/etc/dh/nginx.pem @@ -31,7 +32,7 @@ echo "writing fresh nginx templates..." cp /usr/share/odk/nginx/redirector.conf /etc/nginx/conf.d/redirector.conf CERT_DOMAIN=$( [ "$SSL_TYPE" = "customssl" ] && echo "local" || echo "$DOMAIN") \ -envsubst '$SSL_TYPE $CERT_DOMAIN $DOMAIN $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT' \ +/scripts/envsub.awk \ < /usr/share/odk/nginx/odk.conf.template \ > /etc/nginx/conf.d/odk.conf diff --git a/files/service/scripts/start-odk.sh b/files/service/scripts/start-odk.sh index 1c2471af..3c5391ee 100755 --- a/files/service/scripts/start-odk.sh +++ b/files/service/scripts/start-odk.sh @@ -4,7 +4,7 @@ echo "generating local service configuration.." ENKETO_API_KEY=$(cat /etc/secrets/enketo-api-key) \ BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \ -envsubst '$DOMAIN $BASE_URL $SYSADMIN_EMAIL $ENKETO_API_KEY $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME $DB_SSL $EMAIL_FROM $EMAIL_HOST $EMAIL_PORT $EMAIL_SECURE $EMAIL_IGNORE_TLS $EMAIL_USER $EMAIL_PASSWORD $OIDC_ENABLED $OIDC_ISSUER_URL $OIDC_CLIENT_ID $OIDC_CLIENT_SECRET $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT $S3_SERVER $S3_ACCESS_KEY $S3_SECRET_KEY $S3_BUCKET_NAME' \ +/scripts/envsub.awk \ < /usr/share/odk/config.json.template \ > /usr/odk/config/local.json diff --git a/files/shared/envsub.awk b/files/shared/envsub.awk new file mode 100755 index 00000000..86a4cff8 --- /dev/null +++ b/files/shared/envsub.awk @@ -0,0 +1,39 @@ +#!/usr/bin/mawk -f + +# Safer implemention of envsubst. +# +# Significant differences: +# +# * require curly brackets around values +# * require uppercase +# * throw error if value is not defined, instead of just using empty string +# +# mawk is the awk implementation common to the containers we need to run this +# script in, so we use it here explicitly. + +BEGIN { + errorCount = 0; +} + +{ + while(match($0, /\$\{[A-Z_][A-Z_0-9]*\}/) > 0) { + k = substr($0, RSTART+2, RLENGTH-3); + if(k in ENVIRON) { + v = ENVIRON[k]; + } else { + print "ERR: var not defined on line " NR ": ${" k "}" > "/dev/fd/2"; + ++errorCount; + v = "!!!VALUE-MISSING: " k "!!!" + } + gsub("\$\{" k "\}", v); + } + print $0; +} + +END { + if(errorCount > 0) { + print "" > "/dev/fd/2"; + print errorCount " error(s) found." > "/dev/fd/2"; + exit 1; + } +} diff --git a/nginx.dockerfile b/nginx.dockerfile index b2b94712..e956e8ae 100644 --- a/nginx.dockerfile +++ b/nginx.dockerfile @@ -28,7 +28,9 @@ RUN apt-get update && apt-get install -y netcat-openbsd RUN mkdir -p /usr/share/odk/nginx/ -COPY files/nginx/setup-odk.sh /scripts/ +COPY files/nginx/setup-odk.sh \ + files/shared/envsub.awk \ + /scripts/ RUN chmod +x /scripts/setup-odk.sh COPY files/nginx/redirector.conf /usr/share/odk/nginx/ diff --git a/service.dockerfile b/service.dockerfile index 82d5848f..fadc7fe9 100644 --- a/service.dockerfile +++ b/service.dockerfile @@ -58,6 +58,7 @@ RUN apt-get update \ --fund=false --update-notifier=false COPY server/ ./ +COPY files/shared/envsub.awk /scripts/ COPY files/service/scripts/ ./ COPY files/service/config.json.template /usr/share/odk/ diff --git a/test/envsub/bad-example.in b/test/envsub/bad-example.in new file mode 100644 index 00000000..fd0ed5e5 --- /dev/null +++ b/test/envsub/bad-example.in @@ -0,0 +1,3 @@ +${NOT_DEFINED_1} ${NOT_DEFINED_2} + +${NOT_DEFINED_3} diff --git a/test/envsub/bad-example.stderr.expected b/test/envsub/bad-example.stderr.expected new file mode 100644 index 00000000..29a5842b --- /dev/null +++ b/test/envsub/bad-example.stderr.expected @@ -0,0 +1,5 @@ +ERR: var not defined on line 1: ${NOT_DEFINED_1} +ERR: var not defined on line 1: ${NOT_DEFINED_2} +ERR: var not defined on line 3: ${NOT_DEFINED_3} + +3 error(s) found. diff --git a/test/envsub/bad-example.stdout.expected b/test/envsub/bad-example.stdout.expected new file mode 100644 index 00000000..f0eb1d85 --- /dev/null +++ b/test/envsub/bad-example.stdout.expected @@ -0,0 +1,3 @@ +!!!VALUE-MISSING: NOT_DEFINED_1!!! !!!VALUE-MISSING: NOT_DEFINED_2!!! + +!!!VALUE-MISSING: NOT_DEFINED_3!!! diff --git a/test/envsub/good-example.expected b/test/envsub/good-example.expected new file mode 100644 index 00000000..0295cb85 --- /dev/null +++ b/test/envsub/good-example.expected @@ -0,0 +1,20 @@ +# Alone: +sv_simple + +# end of line +Simple value: sv_simple + +# Missing curlies: +Should not be substituted: $SIMPLE + +# Start of line: +sv_simple and then more content. + +# This is not a sub variable: +$simple + +# Two values on the same line: +first: sub_val_one, second: sub_val_two! + +# Two values on the same line, both repeated: +first: sub_val_one, sub_val_one, sub_val_one, second: sub_val_two sub_val_two! diff --git a/test/envsub/good-example.in b/test/envsub/good-example.in new file mode 100644 index 00000000..99d2f8f6 --- /dev/null +++ b/test/envsub/good-example.in @@ -0,0 +1,20 @@ +# Alone: +${SIMPLE} + +# end of line +Simple value: ${SIMPLE} + +# Missing curlies: +Should not be substituted: $SIMPLE + +# Start of line: +${SIMPLE} and then more content. + +# This is not a sub variable: +$simple + +# Two values on the same line: +first: ${SUBVAL_1}, second: ${SUBVAL_2}! + +# Two values on the same line, both repeated: +first: ${SUBVAL_1}, ${SUBVAL_1}, ${SUBVAL_1}, second: ${SUBVAL_2} ${SUBVAL_2}! diff --git a/test/envsub/run-tests.sh b/test/envsub/run-tests.sh new file mode 100755 index 00000000..a8444ee5 --- /dev/null +++ b/test/envsub/run-tests.sh @@ -0,0 +1,50 @@ +#!/bin/bash -eu +log() { echo >&2 "[test/envsub] $*"; } + +failCount=0 + +log "should correctly substitute provided values" +if diff <( \ + SIMPLE=sv_simple \ + SUBVAL_1=sub_val_one \ + SUBVAL_2=sub_val_two \ + ../../files/shared/envsub.awk \ +< good-example.in +) good-example.expected; then + log " OK" +else + ((++failCount)) + log " FAILED" +fi + +log "should fail when asked to substitute undefined value" +if ! ../../files/shared/envsub.awk <<<"\${NOT_DEFINED}" >/dev/null 2>/dev/null; then + log " OK" +else + ((++failCount)) + log " FAILED" +fi + +log "should log all issues when asked to substitute multiple undefined values" +out="$(mktemp)" +err="$(mktemp)" +if ../../files/shared/envsub.awk < bad-example.in >"$out" 2>"$err"; then + ((++failCount)) + log " FAILED: expected non-zero status code" +elif ! diff "$out" bad-example.stdout.expected; then + ((++failCount)) + log " FAILED: generated stdout did not equal expected output" +elif ! diff "$err" bad-example.stderr.expected; then + echo "err: $err" + ((++failCount)) + log " FAILED: generated stderr did not equal expected output" +else + log " OK" +fi + +if [[ "$failCount" = 0 ]]; then + log "All tests passed OK." +else + log "$failCount TEST(S) FAILED" + exit 1 +fi diff --git a/test/.gitignore b/test/nginx/.gitignore similarity index 100% rename from test/.gitignore rename to test/nginx/.gitignore diff --git a/test/files/nginx-test/acme-challenge b/test/nginx/files/nginx-test/acme-challenge similarity index 100% rename from test/files/nginx-test/acme-challenge rename to test/nginx/files/nginx-test/acme-challenge diff --git a/test/files/nginx-test/http_root/index.html b/test/nginx/files/nginx-test/http_root/index.html similarity index 100% rename from test/files/nginx-test/http_root/index.html rename to test/nginx/files/nginx-test/http_root/index.html diff --git a/test/files/nginx-test/http_root/should-be-cached.txt b/test/nginx/files/nginx-test/http_root/should-be-cached.txt similarity index 100% rename from test/files/nginx-test/http_root/should-be-cached.txt rename to test/nginx/files/nginx-test/http_root/should-be-cached.txt diff --git a/test/files/nginx-test/http_root/version.txt b/test/nginx/files/nginx-test/http_root/version.txt similarity index 100% rename from test/files/nginx-test/http_root/version.txt rename to test/nginx/files/nginx-test/http_root/version.txt diff --git a/test/mock-http-server/.gitignore b/test/nginx/mock-http-server/.gitignore similarity index 100% rename from test/mock-http-server/.gitignore rename to test/nginx/mock-http-server/.gitignore diff --git a/test/mock-http-server/index.js b/test/nginx/mock-http-server/index.js similarity index 100% rename from test/mock-http-server/index.js rename to test/nginx/mock-http-server/index.js diff --git a/test/mock-http-server/package-lock.json b/test/nginx/mock-http-server/package-lock.json similarity index 100% rename from test/mock-http-server/package-lock.json rename to test/nginx/mock-http-server/package-lock.json diff --git a/test/mock-http-server/package.json b/test/nginx/mock-http-server/package.json similarity index 100% rename from test/mock-http-server/package.json rename to test/nginx/mock-http-server/package.json diff --git a/test/mock-http-service.dockerfile b/test/nginx/mock-http-service.dockerfile similarity index 100% rename from test/mock-http-service.dockerfile rename to test/nginx/mock-http-service.dockerfile diff --git a/test/nginx.test.docker-compose.yml b/test/nginx/nginx.test.docker-compose.yml similarity index 66% rename from test/nginx.test.docker-compose.yml rename to test/nginx/nginx.test.docker-compose.yml index 4a863054..832b7cb5 100644 --- a/test/nginx.test.docker-compose.yml +++ b/test/nginx/nginx.test.docker-compose.yml @@ -15,7 +15,7 @@ services: - PORT=8383 nginx: build: - context: .. + context: ../.. dockerfile: nginx.dockerfile args: SKIP_FRONTEND_BUILD: true @@ -24,12 +24,15 @@ services: - enketo environment: - DOMAIN=odk-nginx.example.test - - SENTRY_ORG_SUBDOMAIN=example + - OIDC_ENABLED= + - SENTRY_KEY=example-sentry-key + - SENTRY_ORG_SUBDOMAIN=example-sentry-org-subdomain + - SENTRY_PROJECT=example-sentry-project - SSL_TYPE=selfsign - OIDC_ENABLED=false volumes: - - ../files/nginx/odk.conf.template:/usr/share/odk/nginx/odk.conf.template:ro - - ../files/nginx/client-config.json.template:/usr/share/odk/nginx/client-config.json.template:ro + - ../../files/nginx/odk.conf.template:/usr/share/odk/nginx/odk.conf.template:ro + - ../../files/nginx/client-config.json.template:/usr/share/odk/nginx/client-config.json.template:ro ports: - "9000:80" - "9001:443" diff --git a/test/package-lock.json b/test/nginx/package-lock.json similarity index 100% rename from test/package-lock.json rename to test/nginx/package-lock.json diff --git a/test/package.json b/test/nginx/package.json similarity index 100% rename from test/package.json rename to test/nginx/package.json diff --git a/test/run-tests.sh b/test/nginx/run-tests.sh similarity index 100% rename from test/run-tests.sh rename to test/nginx/run-tests.sh diff --git a/test/test-nginx.js b/test/nginx/test-nginx.js similarity index 100% rename from test/test-nginx.js rename to test/nginx/test-nginx.js From 0dd2323ad7011b9b8c207f9093d09aac7edc0f62 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 10 Dec 2024 07:08:02 +0000 Subject: [PATCH 2/4] use /dev/stderr --- files/shared/envsub.awk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/shared/envsub.awk b/files/shared/envsub.awk index 86a4cff8..30df6d11 100755 --- a/files/shared/envsub.awk +++ b/files/shared/envsub.awk @@ -21,7 +21,7 @@ BEGIN { if(k in ENVIRON) { v = ENVIRON[k]; } else { - print "ERR: var not defined on line " NR ": ${" k "}" > "/dev/fd/2"; + print "ERR: var not defined on line " NR ": ${" k "}" > "/dev/stderr"; ++errorCount; v = "!!!VALUE-MISSING: " k "!!!" } @@ -32,8 +32,8 @@ BEGIN { END { if(errorCount > 0) { - print "" > "/dev/fd/2"; - print errorCount " error(s) found." > "/dev/fd/2"; + print "" > "/dev/stderr"; + print errorCount " error(s) found." > "/dev/stderr"; exit 1; } } From 3e195b0bd884b022cf963ac8ae91e9d45b894239 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 10 Dec 2024 07:16:05 +0000 Subject: [PATCH 3/4] revert whitespace change --- files/nginx/setup-odk.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/files/nginx/setup-odk.sh b/files/nginx/setup-odk.sh index d691247f..ffdccb26 100644 --- a/files/nginx/setup-odk.sh +++ b/files/nginx/setup-odk.sh @@ -1,5 +1,6 @@ #!/bin/bash + echo "writing client config..." if [[ $OIDC_ENABLED != 'true' ]] && [[ $OIDC_ENABLED != 'false' ]]; then echo 'OIDC_ENABLED must be either true or false' From 6ccdb0eabbcd076fe01fa38ccbc1a8dcd3afed64 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 10 Dec 2024 07:16:51 +0000 Subject: [PATCH 4/4] run on redirector.conf (merged from next) --- files/nginx/setup-odk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/nginx/setup-odk.sh b/files/nginx/setup-odk.sh index ffdccb26..e96bcb48 100644 --- a/files/nginx/setup-odk.sh +++ b/files/nginx/setup-odk.sh @@ -39,7 +39,7 @@ fi # start from fresh templates in case ssl type has changed echo "writing fresh nginx templates..." # redirector.conf gets deleted if using upstream SSL so copy it back -envsubst '$DOMAIN' \ +/scripts/envsub.awk \ < /usr/share/odk/nginx/redirector.conf \ > /etc/nginx/conf.d/redirector.conf