Upgrade to Couchbase 4 (and other build changes)

.github/workflows/test.yml

@@ -81,8 +81,6 @@ jobs:
         working-directory: ./nodecg/bundles/ystv-sports-graphics
         run: |
           yarn --immutable --inline-builds
-          mkdir -p scores-src/build/Release/
-          cp node_modules/couchbase/build/Release/couchbase_impl.node scores-src/build/Release/couchbase_impl.node
 
       - name: Run integration tests
         run: nodecg/bundles/ystv-sports-graphics/scripts/test-integration.sh
@@ -138,8 +136,6 @@ jobs:
         working-directory: ./nodecg/bundles/ystv-sports-graphics
         run: |
           yarn --immutable --inline-builds
-          mkdir -p scores-src/build/Release/
-          cp node_modules/couchbase/build/Release/couchbase_impl.node scores-src/build/Release/couchbase_impl.node
 
       - name: Install NodeCG dependencies
         if: matrix.component == 'bundle'

.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch server (built)",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "program": "${workspaceFolder}/scores-src/dist/index.server.js",
+      "cwd": "${workspaceFolder}/scores-src",
+      "outFiles": [
+        "${workspaceFolder}/scores-src/**/*.js"
+      ]
+    }
+  ]
+}

Dockerfile.server

@@ -23,12 +23,13 @@ RUN yarn build:server
 FROM node:16-slim
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
-# needed for libcouchbase
-COPY --from=build /app/node_modules/couchbase/build/Release/couchbase_impl.node /app/addon-build/release/install-root/couchbase_impl.node
+WORKDIR /app
 COPY --from=build /app/scores-src/config /app/config
 COPY --from=build /app/scores-src/package.json /app/
+# couchbase has native dependencies - this'll install the right one for the Dockerfile platform
+# the awk command extracts the installed version, to make sure we download the same one
+RUN npm install argon2@$( npm list argon2 | awk -F@ '/argon2/ { print $2}' ) couchbase@$( npm list couchbase | awk -F@ '/couchbase/ { print $2}' )
 COPY --from=build /app/scores-src/dist/ /app/
-WORKDIR /app
 EXPOSE 8000
 CMD ["--enable-source-maps", "index.server.js"]
 

scores-src/esbuild.config.js

@@ -1,40 +1,3 @@
-/* eslint-disable */
-const nativeNodeModulesPlugin = {
-  name: "native-node-modules",
-  setup(build) {
-    // If a ".node" file is imported within a module in the "file" namespace, resolve
-    // it to an absolute path and put it into the "node-file" virtual namespace.
-    build.onResolve({ filter: /\.node$/, namespace: "file" }, (args) => ({
-      path: require.resolve(args.path, { paths: [args.resolveDir] }),
-      namespace: "node-file",
-    }));
-
-    // Files in the "node-file" virtual namespace call "require()" on the
-    // path from esbuild of the ".node" file in the output directory.
-    build.onLoad({ filter: /.*/, namespace: "node-file" }, (args) => ({
-      contents: `
-          import path from ${JSON.stringify(args.path)}
-          try { module.exports = require(path) }
-          catch {}
-        `,
-    }));
-
-    // If a ".node" file is imported within a module in the "node-file" namespace, put
-    // it in the "file" namespace where esbuild's default loading behavior will handle
-    // it. It is already an absolute path since we resolved it to one above.
-    build.onResolve({ filter: /\.node$/, namespace: "node-file" }, (args) => ({
-      path: args.path,
-      namespace: "file",
-    }));
-
-    // Tell esbuild's default loading behavior to use the "file" loader for
-    // these ".node" files.
-    let opts = build.initialOptions;
-    opts.loader = opts.loader || {};
-    opts.loader[".node"] = "file";
-  },
-};
-
 module.exports = {
   entryPoints: [
     "./src/server/index.server.ts",
@@ -45,6 +8,9 @@ module.exports = {
   treeShaking: true,
   sourcemap: true,
   bundle: true,
-  plugins: [nativeNodeModulesPlugin],
+  loader: {
+    ".node": "copy",
+  },
   outdir: "./dist",
+  external: ["argon2", "couchbase"],
 };

scores-src/package.json

@@ -8,7 +8,7 @@
     "dev:server": "node startDevServer.js",
     "dev:server:typecheck": "cd src/server && tsc -w --preserveWatchOutput",
     "dev:client": "vite dev",
-    "build": "rm -rf dist && yarn build:client && yarn build:server",
+    "build": "rm -rf dist && run-p build:client build:server",
     "build:server": "node buildProdServer.js",
     "build:client": "vite build",
     "prod": "concurrently -k -n server,client npm:prod:server npm:prod:client",
@@ -36,7 +36,7 @@
     "babel-jest": "^27.5.1",
     "concurrently": "^7.0.0",
     "del": "^6.1.1",
-    "esbuild": "^0.14.23",
+    "esbuild": "^0.19.10",
     "happy-dom": "^8.1.1",
     "http-proxy": "^1.18.1",
     "isomorphic-fetch": "^3.0.0",
@@ -80,14 +80,14 @@
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^8.3.4",
     "@types/winston": "^2.4.4",
-    "argon2": "^0.28.5",
+    "argon2": "^0.31.2",
     "body-parser": "^1.20.0",
     "conf": "^10.1.1",
     "config": "^3.3.7",
     "convict": "^6.2.1",
     "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
-    "couchbase": "3.2.4",
+    "couchbase": "^4.2.8",
     "dayjs": "^1.11.1",
     "dotenv-flow": "^3.2.0",
     "express": "^4.17.3",

scores-src/src/server/__mocks__/db.ts

@@ -13,23 +13,25 @@ import {
   RemoveOptions,
   GetAndLockOptions,
   UnlockOptions,
+  Cas,
 } from "couchbase";
 import { cloneDeep, get, isEqual, set } from "lodash-es";
 import binding from "couchbase/dist/binding";
+import { CouchbaseCas } from "../dbHelpers";
 
 interface Rec {
   value: unknown;
-  cas: number;
-  oldCas?: number;
+  cas: string;
+  oldCas?: string;
 }
 
-function newCas(): number {
-  return Math.round(Math.random() * 1_000_000_000);
+function newCas(): string {
+  return Math.round(Math.random() * 1_000_000_000).toString(10);
 }
 
 class MemDBError extends Error {}
 
-const LOCKED_CAS = 0xff;
+const LOCKED_CAS = "-1";
 
 export class InMemoryDB {
   private collections: Map<string, Map<string, Rec>> = new Map();
@@ -75,7 +77,9 @@ export class InMemoryDB {
         }
         return {
           content: cloneDeep(val.value),
-          cas: val.cas,
+          cas: CouchbaseCas.from(val.cas),
+          value: undefined,
+          expiry: undefined,
         } as GetResult;
       },
       /**
@@ -97,13 +101,13 @@ export class InMemoryDB {
         c.set(key, { ...val, cas: LOCKED_CAS, oldCas });
         return { ...val, cas: oldCas };
       },
-      async unlock(key: string, cas: number, options?: UnlockOptions) {
+      async unlock(key: string, cas: Cas, options?: UnlockOptions) {
         const val = c.get(key);
         if (!val) {
           console.warn("Document not found: ", key);
           throw new DocumentNotFoundError(new MemDBError(key));
         }
-        if (val.oldCas !== cas) {
+        if (val.oldCas !== cas.toString()) {
           throw new CasMismatchError(
             new MemDBError("Unlock CAS mismatch: " + key)
           );
@@ -123,7 +127,7 @@ export class InMemoryDB {
         }
         if (options) {
           if (options.cas) {
-            if (c.get(key)?.cas !== options.cas) {
+            if (c.get(key)?.cas !== options.cas.toString()) {
               throw new CasMismatchError(
                 new MemDBError(`expected ${c.get(key)?.cas} got ${options.cas}`)
               );
@@ -148,7 +152,7 @@ export class InMemoryDB {
         }
         if (options) {
           if (options.cas) {
-            if (c.get(key)?.cas !== options.cas) {
+            if (c.get(key)?.cas !== options.cas.toString()) {
               throw new CasMismatchError(
                 new MemDBError(`expected ${c.get(key)?.cas} got ${options.cas}`)
               );
@@ -167,7 +171,7 @@ export class InMemoryDB {
         }
         if (options) {
           if (options.cas) {
-            if (c.get(key)?.cas !== options.cas) {
+            if (c.get(key)?.cas !== options.cas.toString()) {
               throw new CasMismatchError(
                 new MemDBError(`expected ${c.get(key)?.cas} got ${options.cas}`)
               );
@@ -178,10 +182,10 @@ export class InMemoryDB {
         let val = c.get(key)!.value as any;
         for (const op of specs) {
           switch (op._op) {
-            case binding.LCBX_SDCMD_ARRAY_ADD_LAST:
+            case binding.protocol_subdoc_opcode.array_push_last:
               val = [...(val as unknown[]), JSON.parse(op._data)];
               break;
-            case binding.LCBX_SDCMD_REMOVE:
+            case binding.protocol_subdoc_opcode.remove:
               if (op._path[op._path.length - 1] == "]") {
                 // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                 const [_, arrayPath, index] = /(.*?)\[([0-9-]+)\]$/.exec(
@@ -201,7 +205,7 @@ export class InMemoryDB {
                 );
               }
               break;
-            case binding.LCBX_SDCMD_ARRAY_INSERT:
+            case binding.protocol_subdoc_opcode.array_insert:
               if (op._path[op._path.length - 1] == "]") {
                 // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                 const [_, arrayPath, index] = /(.*?)\[([0-9-]+)\]$/.exec(
@@ -221,7 +225,7 @@ export class InMemoryDB {
                 );
               }
               break;
-            case binding.LCBX_SDCMD_REPLACE:
+            case binding.protocol_subdoc_opcode.replace:
               if (op._path[op._path.length - 1] == "]") {
                 // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                 const [_, arrayPath, index] = /(.*?)\[([0-9-]+)\]$/.exec(

scores-src/src/server/dbHelpers.ts

@@ -0,0 +1,24 @@
+import { Cas } from "couchbase";
+
+// https://issues.couchbase.com/browse/JSCBC-1164
+export class CouchbaseCas implements Cas {
+  private readonly cas: string;
+
+  constructor(casString: string) {
+    this.cas = casString;
+  }
+
+  static from(casString: string | number) {
+    return new CouchbaseCas(
+      typeof casString === "number" ? casString.toString() : casString
+    );
+  }
+
+  toJSON() {
+    return this.cas;
+  }
+
+  toString() {
+    return this.cas;
+  }
+}

scores-src/src/server/eventTypeRoutes.ts

@@ -8,6 +8,7 @@ import {
   DocumentNotFoundError,
   MutateInSpec,
   QueryScanConsistency,
+  Cas,
 } from "couchbase";
 import { dispatchChangeToEvent, resync } from "./updatesRepo";
 import {
@@ -35,6 +36,7 @@ import {
 import { doUpdate as updateTournamentSummary } from "./updateTournamentSummary.job";
 import { cloneDeep, identity, isEqual, omit, pickBy } from "lodash-es";
 import { leagueKey } from "./leagueRoutes";
+import { CouchbaseCas } from "./dbHelpers";
 
 export function makeEventAPIFor<
   TState extends BaseEventStateType,
@@ -262,7 +264,7 @@ export function makeEventAPIFor<
         historyKey(league, id),
         [MutateInSpec.arrayAppend("", editAction)],
         {
-          cas: cas ?? history.cas,
+          cas: cas ? CouchbaseCas.from(cas) : history.cas,
         }
       );
 

scores-src/src/server/eventsRoutes.ts

@@ -13,6 +13,7 @@ import { ensure } from "./errs";
 import { BadRequest } from "http-errors";
 import { doUpdate as updateTournamentSummary } from "./updateTournamentSummary.job";
 import { getLogger } from "./loggingSetup";
+import { CouchbaseCas } from "./dbHelpers";
 
 export function createEventsRouter() {
   const router = Router();
@@ -134,7 +135,7 @@ export function createEventsRouter() {
         "league",
       ]).validate(inputData, { abortEarly: false, stripUnknown: true });
       await DB.collection("_default").replace(metaKey(league, type, id), val, {
-        cas: cas,
+        cas: cas ? CouchbaseCas.from(cas) : undefined,
       });
       res.status(200).json(val);
     })

scores-src/src/server/userManagementRoutes.ts

@@ -93,7 +93,7 @@ export function createUserManagementRouter() {
       };
       const data = await DB.collection("_default").getAndLock(id, 10);
       try {
-        if (payload._cas && data.cas != payload._cas) {
+        if (payload._cas && data.cas.toString() != payload._cas) {
           throw new Conflict("someone else has updated this user");
         }
         newData.passwordHash = data.content.passwordHash;

scripts/test-e2e.sh

@@ -35,10 +35,12 @@ fi
 if ! curl -fs -o /dev/null http://localhost:8000/healthz; then
   echo "Starting scores server..."
   yarn prod:server >"$SCRIPT_DIR/../test-server.log" 2>&1 &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:8000/healthz
 fi
 if ! curl -fs -o /dev/null http://localhost:3000; then
   echo "Starting scores client..."
   yarn prod:client >/dev/null &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:3000
 fi
 
 popd || exit 1
@@ -69,6 +71,7 @@ EOF
 
   echo "Starting NodeCG..."
   node "$SCRIPT_DIR/../../../index.js" >"$SCRIPT_DIR/../test-nodecg.log" 2>&1 &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:9090
   popd || exit 1
 fi
 

scripts/test-integration.sh

@@ -17,6 +17,7 @@ if ! curl -fs -o /dev/null http://localhost:8000/healthz; then
 
   echo "Starting scores server..."
   yarn prod:server >"$SCRIPT_DIR/../test-server.log" 2>&1 &
+  curl --retry 30 --retry-delay 0 --retry-all-errors -fs -o /dev/null http://localhost:8000/healthz
 fi
 
 # Run tests