Upgrade to Couchbase 4 (and other build changes)

.github/workflows/test.yml

@@ -81,8 +81,6 @@ jobs:
         working-directory: ./nodecg/bundles/ystv-sports-graphics
         run: |
           yarn --immutable --inline-builds
-          mkdir -p scores-src/build/Release/
-          cp node_modules/couchbase/build/Release/couchbase_impl.node scores-src/build/Release/couchbase_impl.node
 
       - name: Run integration tests
         run: nodecg/bundles/ystv-sports-graphics/scripts/test-integration.sh
@@ -138,8 +136,6 @@ jobs:
         working-directory: ./nodecg/bundles/ystv-sports-graphics
         run: |
           yarn --immutable --inline-builds
-          mkdir -p scores-src/build/Release/
-          cp node_modules/couchbase/build/Release/couchbase_impl.node scores-src/build/Release/couchbase_impl.node
 
       - name: Install NodeCG dependencies
         if: matrix.component == 'bundle'

.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch server (built)",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "program": "${workspaceFolder}/scores-src/dist/index.server.js",
+      "cwd": "${workspaceFolder}/scores-src",
+      "outFiles": [
+        "${workspaceFolder}/scores-src/**/*.js"
+      ]
+    }
+  ]
+}

Dockerfile.server

@@ -23,12 +23,13 @@ RUN yarn build:server
 FROM node:16-slim
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
-# needed for libcouchbase
-COPY --from=build /app/node_modules/couchbase/build/Release/couchbase_impl.node /app/addon-build/release/install-root/couchbase_impl.node
+WORKDIR /app
 COPY --from=build /app/scores-src/config /app/config
 COPY --from=build /app/scores-src/package.json /app/
+# couchbase has native dependencies - this'll install the right one for the Dockerfile platform
+# the awk command extracts the installed version, to make sure we download the same one
+RUN npm install argon2@$( npm list argon2 | awk -F@ '/argon2/ { print $2}' ) couchbase@$( npm list couchbase | awk -F@ '/couchbase/ { print $2}' )
 COPY --from=build /app/scores-src/dist/ /app/
-WORKDIR /app
 EXPOSE 8000
 CMD ["--enable-source-maps", "index.server.js"]
 

scores-src/esbuild.config.js

@@ -1,40 +1,3 @@
-/* eslint-disable */
-const nativeNodeModulesPlugin = {
-  name: "native-node-modules",
-  setup(build) {
-    // If a ".node" file is imported within a module in the "file" namespace, resolve
-    // it to an absolute path and put it into the "node-file" virtual namespace.
-    build.onResolve({ filter: /\.node$/, namespace: "file" }, (args) => ({
-      path: require.resolve(args.path, { paths: [args.resolveDir] }),
-      namespace: "node-file",
-    }));
-
-    // Files in the "node-file" virtual namespace call "require()" on the
-    // path from esbuild of the ".node" file in the output directory.
-    build.onLoad({ filter: /.*/, namespace: "node-file" }, (args) => ({
-      contents: `
-          import path from ${JSON.stringify(args.path)}
-          try { module.exports = require(path) }
-          catch {}
-        `,
-    }));
-
-    // If a ".node" file is imported within a module in the "node-file" namespace, put
-    // it in the "file" namespace where esbuild's default loading behavior will handle
-    // it. It is already an absolute path since we resolved it to one above.
-    build.onResolve({ filter: /\.node$/, namespace: "node-file" }, (args) => ({
-      path: args.path,
-      namespace: "file",
-    }));
-
-    // Tell esbuild's default loading behavior to use the "file" loader for
-    // these ".node" files.
-    let opts = build.initialOptions;
-    opts.loader = opts.loader || {};
-    opts.loader[".node"] = "file";
-  },
-};
-
 module.exports = {
   entryPoints: [
     "./src/server/index.server.ts",
@@ -45,6 +8,9 @@ module.exports = {
   treeShaking: true,
   sourcemap: true,
   bundle: true,
-  plugins: [nativeNodeModulesPlugin],
+  loader: {
+    ".node": "copy",
+  },
   outdir: "./dist",
+  external: ["argon2", "couchbase"],
 };

scores-src/package.json

@@ -8,7 +8,7 @@
     "dev:server": "node startDevServer.js",
     "dev:server:typecheck": "cd src/server && tsc -w --preserveWatchOutput",
     "dev:client": "vite dev",
-    "build": "rm -rf dist && yarn build:client && yarn build:server",
+    "build": "rm -rf dist && run-p build:client build:server",
     "build:server": "node buildProdServer.js",
     "build:client": "vite build",
     "prod": "concurrently -k -n server,client npm:prod:server npm:prod:client",
@@ -36,7 +36,7 @@
     "babel-jest": "^27.5.1",
     "concurrently": "^7.0.0",
     "del": "^6.1.1",
-    "esbuild": "^0.14.23",
+    "esbuild": "^0.19.10",
     "happy-dom": "^8.1.1",
     "http-proxy": "^1.18.1",
     "isomorphic-fetch": "^3.0.0",
@@ -80,14 +80,14 @@
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^8.3.4",
     "@types/winston": "^2.4.4",
-    "argon2": "^0.28.5",
+    "argon2": "^0.31.2",
     "body-parser": "^1.20.0",
     "conf": "^10.1.1",
     "config": "^3.3.7",
     "convict": "^6.2.1",
     "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
-    "couchbase": "3.2.4",
+    "couchbase": "^4.2.8",
     "dayjs": "^1.11.1",
     "dotenv-flow": "^3.2.0",
     "express": "^4.17.3",

scripts/test-e2e.sh

@@ -35,10 +35,12 @@ fi
 if ! curl -fs -o /dev/null http://localhost:8000/healthz; then
   echo "Starting scores server..."
   yarn prod:server >"$SCRIPT_DIR/../test-server.log" 2>&1 &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:8000/healthz
 fi
 if ! curl -fs -o /dev/null http://localhost:3000; then
   echo "Starting scores client..."
   yarn prod:client >/dev/null &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:3000
 fi
 
 popd || exit 1
@@ -69,6 +71,7 @@ EOF
 
   echo "Starting NodeCG..."
   node "$SCRIPT_DIR/../../../index.js" >"$SCRIPT_DIR/../test-nodecg.log" 2>&1 &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:9090
   popd || exit 1
 fi
 

scripts/test-integration.sh

@@ -17,6 +17,7 @@ if ! curl -fs -o /dev/null http://localhost:8000/healthz; then
 
   echo "Starting scores server..."
   yarn prod:server >"$SCRIPT_DIR/../test-server.log" 2>&1 &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:8000/healthz
 fi
 
 # Run tests