feat(root, api-service, worker, ws, webhook): migrate from aws ssm to…

… aws secret manager (novuhq#7473)
ayasofyazilim-clomerce · Jan 9, 2025 · fd3cff6 · fd3cff6
1 parent 5e9772c
commit fd3cff6
Show file tree

Hide file tree

Showing 19 changed files with 1,930 additions and 1,189 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -718,7 +718,8 @@
     "touchpoint",
     "Angularjs",
     "navigatable",
-    "facated"
+    "facated",
+    "dotenvcreate"
   ],
   "flagWords": [],
   "patterns": [

diff --git a/.github/actions/docker/build-api/action.yml b/.github/actions/docker/build-api/action.yml
@@ -74,6 +74,7 @@ runs:
           --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
       run: |
         set -x
+        cp scripts/dotenvcreate.mjs apps/api/src/dotenvcreate.mjs
         cd apps/api && pnpm run docker:build
 
     - name: Tag and test

diff --git a/.github/actions/docker/build-worker/action.yml b/.github/actions/docker/build-worker/action.yml
@@ -74,6 +74,7 @@ runs:
           --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
       run: |
         set -x
+        cp scripts/dotenvcreate.mjs apps/worker/src/dotenvcreate.mjs
         cd apps/worker && pnpm run docker:build
 
     - name: Tag and test

diff --git a/.github/actions/setup-project/action.yml b/.github/actions/setup-project/action.yml
@@ -41,7 +41,7 @@ runs:
       if: ${{ inputs.slim == 'false' }}
       uses: supercharge/[email protected]
       with:
-        mongodb-version: 5.0.29
+        mongodb-version: 8.0
 
     - name: 🛟 Install dependencies
       shell: bash

diff --git a/.github/workflows/dev-deploy-ws.yml b/.github/workflows/dev-deploy-ws.yml
@@ -109,6 +109,7 @@ jobs:
             --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
         run: |
           set -x
+          cp scripts/dotenvcreate.mjs apps/ws/src/dotenvcreate.mjs
           cd apps/ws && pnpm run docker:build
 
       - name: Tag and test

diff --git a/.github/workflows/prod-deploy-api.yml b/.github/workflows/prod-deploy-api.yml
@@ -70,6 +70,7 @@ jobs:
             --platform=linux/amd64
             --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
         run: |
+          cp scripts/dotenvcreate.mjs apps/api/src/dotenvcreate.mjs
           cd apps/api && pnpm --silent --workspace-root pnpm-context -- apps/api/Dockerfile | BULL_MQ_PRO_NPM_TOKEN=${BULL_MQ_PRO_NPM_TOKEN} docker buildx build --secret id=BULL_MQ_PRO_NPM_TOKEN --build-arg PACKAGE_PATH=apps/api - -t novu-api --load $DOCKER_BUILD_ARGUMENTS
           docker tag novu-api $REGISTRY/$REPOSITORY:latest
           docker tag novu-api $REGISTRY/$REPOSITORY:prod

diff --git a/.github/workflows/prod-deploy-worker.yml b/.github/workflows/prod-deploy-worker.yml
@@ -70,6 +70,7 @@ jobs:
             --platform=linux/amd64
             --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
         run: |
+          cp scripts/dotenvcreate.mjs apps/worker/src/dotenvcreate.mjs
           cd apps/worker && pnpm --silent --workspace-root pnpm-context -- apps/worker/Dockerfile | BULL_MQ_PRO_NPM_TOKEN=${BULL_MQ_PRO_NPM_TOKEN} docker buildx build --secret id=BULL_MQ_PRO_NPM_TOKEN --build-arg PACKAGE_PATH=apps/worker - -t novu-worker --load $DOCKER_BUILD_ARGUMENTS
           docker tag novu-worker $REGISTRY/$REPOSITORY:latest
           docker tag novu-worker $REGISTRY/$REPOSITORY:prod

diff --git a/.github/workflows/prod-deploy-ws.yml b/.github/workflows/prod-deploy-ws.yml
@@ -66,6 +66,7 @@ jobs:
             --platform=linux/amd64
             --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
         run: |
+          cp scripts/dotenvcreate.mjs apps/ws/src/dotenvcreate.mjs
           BULL_MQ_PRO_NPM_TOKEN=${BULL_MQ_PRO_NPM_TOKEN} docker buildx build --secret id=BULL_MQ_PRO_NPM_TOKEN -t $REGISTRY/$REPOSITORY:$IMAGE_TAG --load -f apps/ws/Dockerfile . $DOCKER_BUILD_ARGUMENTS
           docker run --network=host --name api -dit --env NODE_ENV=test $REGISTRY/$REPOSITORY:$IMAGE_TAG
           docker run --network=host appropriate/curl --retry 10 --retry-delay 5 --retry-connrefused http://127.0.0.1:1340/v1/health-check | grep 'ok'

diff --git a/.github/workflows/reusable-docker.yml b/.github/workflows/reusable-docker.yml
@@ -116,6 +116,7 @@ jobs:
             --platform=linux/amd64 --provenance=false
             --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
         run: |
+          cp scripts/dotenvcreate.mjs $PROJECT_PATH/src/dotenvcreate.mjs
           cd $PROJECT_PATH && npm run docker:build
           docker tag $LOCAL_TAG $REGISTRY/$REPOSITORY:$IMAGE_TAG
           docker tag $LOCAL_TAG $REGISTRY/$REPOSITORY:$ENV_TAG
@@ -137,6 +138,7 @@ jobs:
             --platform=linux/amd64
             --output=type=image,name=$REGISTRY/$REPOSITORY,push-by-digest=true,name-canonical=true
         run: |
+          cp scripts/dotenvcreate.mjs $PROJECT_PATH/src/dotenvcreate.mjs
           cd $PROJECT_PATH && npm run docker:build
           docker tag $LOCAL_TAG $REGISTRY/$REPOSITORY:$IMAGE_TAG
           docker tag $LOCAL_TAG $REGISTRY/$REPOSITORY:$ENV_TAG

diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
@@ -31,6 +31,7 @@ RUN --mount=type=secret,id=BULL_MQ_PRO_NPM_TOKEN,uid=1000 export BULL_MQ_PRO_NPM
 
 WORKDIR /usr/src/app/apps/api
 
+RUN cp src/dotenvcreate.mjs dist/dotenvcreate.mjs
 RUN cp src/.example.env dist/.env
 RUN cp src/.env.test dist/.env.test
 RUN cp src/.env.development dist/.env.development
@@ -69,4 +70,4 @@ RUN --mount=type=cache,id=pnpm-store-api,target=/root/.pnpm-store\
 ENV NEW_RELIC_NO_CONFIG_FILE=true
 
 WORKDIR /usr/src/app/apps/api
-CMD [ "pm2-runtime","start", "dist/main.js" ]
+ENTRYPOINT [ "sh", "-c", "node dist/dotenvcreate.mjs -s=novu/api -r=$NOVU_REGION -e=$NOVU_ENTERPRISE -v=$NODE_ENV && pm2-runtime start dist/main.js" ]
diff --git a/apps/api/package.json b/apps/api/package.json
@@ -31,6 +31,7 @@
     "admin:remove-organization": "cross-env NODE_ENV=local MIGRATION=true ts-node --transpileOnly ./admin/remove-organization.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.716.0",
     "@godaddy/terminus": "^4.12.1",
     "@google-cloud/storage": "^6.2.3",
     "@maily-to/render": "^0.0.17",
@@ -60,6 +61,8 @@
     "@sentry/tracing": "^7.40.0",
     "@types/newrelic": "^9.14.6",
     "@upstash/ratelimit": "^0.4.4",
+    "ajv": "^8.12.0",
+    "ajv-formats": "^2.1.1",
     "axios": "^1.6.8",
     "bcrypt": "^5.0.0",
     "body-parser": "^1.20.0",
@@ -102,8 +105,6 @@
     "twilio": "^4.14.1",
     "uuid": "^8.3.2",
     "zod": "^3.23.8",
-    "ajv": "^8.12.0",
-    "ajv-formats": "^2.1.1",
     "zod-to-json-schema": "^3.23.3"
   },
   "devDependencies": {

diff --git a/apps/webhook/Dockerfile b/apps/webhook/Dockerfile
@@ -13,7 +13,11 @@ RUN --mount=type=cache,id=pnpm-store-webhook,target=/root/.pnpm-store \
 
 # Set the working directory to the webhook app and copy example environment file
 WORKDIR /usr/src/app/apps/webhook
+RUN cp src/dotenvcreate.mjs dist/dotenvcreate.mjs
 RUN cp src/.example.env dist/.env
+RUN cp src/.env.test dist/.env.test
+RUN cp src/.env.development dist/.env.development
+RUN cp src/.env.production dist/.env.production
 
 # Set the working directory to the root of the app
 WORKDIR /usr/src/app
@@ -47,4 +51,4 @@ RUN --mount=type=cache,id=pnpm-store-webhook,target=/root/.pnpm-store \
 
 # Set the working directory to the webhook app and start the application using pm2-runtime
 WORKDIR /usr/src/app/apps/webhook
-CMD [ "pm2-runtime", "start", "dist/main.js" ]
+ENTRYPOINT [ "sh", "-c", "node dist/dotenvcreate.mjs -s=novu/webhook -r=$NOVU_REGION -e=$NOVU_ENTERPRISE -v=$NODE_ENV && pm2-runtime start dist/main.js" ]
diff --git a/apps/webhook/package.json b/apps/webhook/package.json
@@ -21,6 +21,7 @@
     "test:e2e": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true mocha --timeout 10000 --require ts-node/register --exit --file e2e/setup.ts e2e/**/*.e2e.ts src/**/*.e2e.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.716.0",
     "@nestjs/axios": "3.0.3",
     "@nestjs/common": "10.4.1",
     "@nestjs/core": "10.4.1",
@@ -48,7 +49,8 @@
     "newrelic": "^12.8.1",
     "reflect-metadata": "0.2.2",
     "rimraf": "^3.0.2",
-    "rxjs": "7.8.1"
+    "rxjs": "7.8.1",
+    "yargs": "^17.7.2"
   },
   "devDependencies": {
     "@nestjs/cli": "10.4.5",

diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile
@@ -31,6 +31,7 @@ RUN --mount=type=secret,id=BULL_MQ_PRO_NPM_TOKEN,uid=1000 export BULL_MQ_PRO_NPM
 
 WORKDIR /usr/src/app/apps/worker
 
+RUN cp src/dotenvcreate.mjs dist/dotenvcreate.mjs
 RUN cp src/.example.env dist/.env
 RUN cp src/.env.test dist/.env.test
 RUN cp src/.env.development dist/.env.development
@@ -69,4 +70,4 @@ RUN --mount=type=cache,id=pnpm-store-worker,target=/root/.pnpm-store\
 ENV NEW_RELIC_NO_CONFIG_FILE=true
 
 WORKDIR /usr/src/app/apps/worker
-CMD [ "pm2-runtime","start", "dist/main.js" ]
+ENTRYPOINT [ "sh", "-c", "node dist/dotenvcreate.mjs -s=novu/worker -r=$NOVU_REGION -e=$NOVU_ENTERPRISE -v=$NODE_ENV && pm2-runtime start dist/main.js" ]
diff --git a/apps/worker/package.json b/apps/worker/package.json
@@ -22,6 +22,7 @@
     "test:e2e": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true mocha --timeout 10000 --require ts-node/register --exit --file e2e/setup.ts src/**/*.e2e.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.716.0",
     "@nestjs/axios": "3.0.3",
     "@nestjs/common": "10.4.1",
     "@nestjs/core": "10.4.1",

diff --git a/apps/ws/Dockerfile b/apps/ws/Dockerfile
@@ -39,9 +39,10 @@ RUN --mount=type=secret,id=BULL_MQ_PRO_NPM_TOKEN,uid=1000 export BULL_MQ_PRO_NPM
 
 WORKDIR /usr/src/app/apps/ws
 
+RUN cp src/dotenvcreate.mjs dist/dotenvcreate.mjs
 RUN cp src/.example.env dist/.env
 RUN cp src/.env.test dist/.env.test
 RUN cp src/.env.development dist/.env.development
 RUN cp src/.env.production dist/.env.production
 
-CMD [ "pm2-runtime", "dist/main.js" ]
+ENTRYPOINT [ "sh", "-c", "node dist/dotenvcreate.mjs -s=novu/ws -r=$NOVU_REGION -e=$NOVU_ENTERPRISE -v=$NODE_ENV && pm2-runtime start dist/main.js" ]
diff --git a/apps/ws/package.json b/apps/ws/package.json
@@ -20,6 +20,7 @@
     "test": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true mocha --timeout 10000 --require ts-node/register --exit --file e2e/setup.ts './src/**/*.spec.ts'"
   },
   "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.716.0",
     "@godaddy/terminus": "^4.3.1",
     "@nestjs/common": "10.4.1",
     "@nestjs/core": "10.4.1",
@@ -36,8 +37,8 @@
     "@novu/testing": "workspace:*",
     "@sentry/browser": "^8.33.1",
     "@sentry/hub": "^7.114.0",
-    "@sentry/node": "^8.33.1",
     "@sentry/nestjs": "^8.33.1",
+    "@sentry/node": "^8.33.1",
     "@sentry/profiling-node": "^8.33.1",
     "@sentry/tracing": "^7.40.0",
     "@socket.io/admin-ui": "^0.5.1",