chore: replace docker shell script with js CLI commands and use for dev and test environments

packages/api/docker/docker-compose-volumes.yml

@@ -1,11 +1,11 @@
 version: '3.6'
-services:
-  ipfs:
-    volumes:
-      - ./compose/ipfs:/data/ipfs
-  cluster:
-    volumes:
-      - ./compose/cluster:/data/ipfs-cluster
-  minio:
-    volumes:
-      - ./compose/minio:/data/minio
+
+volumes:
+  nftstorage-db-data:
+    external: true
+  nftstorage-ipfs-data:
+    external: true
+  nftstorage-cluster-data:
+    external: true
+  nftstorage-minio-data:
+    external: true

packages/api/docker/docker-compose.yml

@@ -25,13 +25,17 @@ services:
           memory: 1G
     ports:
       - 5432
+    volumes:
+      - nftstorage-db-data:/var/lib/postgresql/data
     environment:
       POSTGRES_DB: postgres
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_PORT: 5432
   ipfs:
     image: ipfs/go-ipfs:v0.10.0 # update this when go-ipfs M1 macs https://github.com/ipfs/go-ipfs/issues/8645
+    volumes:
+      - nftstorage-ipfs-data:/data/ipfs
 
   cluster:
     image: ipfs/ipfs-cluster:v1.0.0-rc4
@@ -49,12 +53,22 @@ services:
       CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery
     ports:
       - 9094
+    volumes:
+      - nftstorage-cluster-data:/data/ipfs-cluster
   minio:
     image: quay.io/minio/minio
     command: server /data/minio --console-address ":9001"
     environment:
       MINIO_ROOT_USER: minioadmin
       MINIO_ROOT_PASSWORD: minioadmin
+    volumes:
+      - nftstorage-minio-data:/data/minio
     ports:
       - 9000
       - 9001
+
+volumes:
+  nftstorage-db-data:
+  nftstorage-ipfs-data:
+  nftstorage-cluster-data:
+  nftstorage-minio-data:

packages/api/package.json

@@ -7,11 +7,10 @@
   "main": "dist/worker.js",
   "scripts": {
     "deploy": "wrangler publish --env production",
-    "predev": "./scripts/cli.js services start && ./scripts/cli.js db-sql --cargo --testing --reset && ./scripts/cli.js minio bucket create dotstorage-dev-0",
-    "dev": "miniflare dist/worker.js --watch --debug --env ../../.env",
-    "dev:persist": "PERSIST_VOLUMES=true npm run dev",
-    "build": "scripts/cli.js build",
-    "test": "./docker/run-with-dependencies.sh ./scripts/run-test.sh",
+    "dev": "npm run build && ./scripts/cli.js run dev",
+    "dev:persist": "npm run build && ./scripts/cli.js run dev --persistent",
+    "build": "./scripts/cli.js build",
+    "test": "./scripts/cli.js build --env=test && scripts/cli.js run test",
     "db-types": "./scripts/cli.js db-types"
   },
   "author": "Hugo Dias <[email protected]> (hugodias.me)",

packages/api/scripts/cli.js

@@ -9,14 +9,12 @@ import Sentry from '@sentry/cli'
 import { createRequire } from 'module'
 // @ts-ignore
 import git from 'git-rev-sync'
-import {
-  servicesStartCmd,
-  servicesStopCmd,
-  servicesPullCmd,
-} from './cmds/services.js'
+import { servicesExecCmd, servicesPullCmd } from './cmds/services.js'
 import { dbSqlCmd } from './cmds/db-sql.js'
 import { dbTypesCmd } from './cmds/db-types.js'
 import { minioBucketCreateCmd, minioBucketRemoveCmd } from './cmds/minio.js'
+import { runTestSuiteCmd } from './cmds/run-test.js'
+import { runDevServerCmd } from './cmds/run-dev.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const require = createRequire(__dirname)
@@ -104,19 +102,12 @@ prog
       process.exit(1)
     }
   })
-  .command('services start')
+  .command('services exec [command]')
   .describe(
-    'Run docker compose to setup Cluster, PostgreSQL, PostgREST and Minio'
+    'Run docker compose to setup Cluster, PostgreSQL, PostgREST and Minio. Executes your command while services are running, then tears down the docker env.'
   )
-  .option('--project', 'Project name', 'nft-storage-dev')
-  .action(servicesStartCmd)
-  .command('services stop')
-  .describe(
-    'Run docker compose to setup Cluster, PostgreSQL, PostgREST and Minio'
-  )
-  .option('--project', 'Project name', 'nft-storage-dev')
-  .option('--clean', 'Clean all dockers artifacts', false)
-  .action(servicesStopCmd)
+  .option('--persistent', 'Whether to enable persistent data volumes', false)
+  .action(servicesExecCmd)
   .command('services pull')
   .describe('pull and build all docker images used for dev/test')
   .action(servicesPullCmd)
@@ -135,5 +126,28 @@ prog
   .command('minio bucket remove <name>')
   .describe('Remove a bucket, automatically removing all contents')
   .action(minioBucketRemoveCmd)
+  .command('run test')
+  .describe(
+    'Run the test suite. Any unrecognized positional args will be passed on to the test runner.'
+  )
+  .option(
+    '--services',
+    'Run service dependencies using docker compose and cleanup after the dev server exits. Set to false if running in a custom environment',
+    true
+  )
+  .action(runTestSuiteCmd)
+  .command('run dev')
+  .describe('Run the development API server using miniflare')
+  .option(
+    '--services',
+    'Run service dependencies using docker compose and cleanup after the dev server exits. Set to false if running in a custom environment',
+    true
+  )
+  .option(
+    '--persistent',
+    'Whether to enable persistent data volumes. Only has an effect when --services=true',
+    false
+  )
+  .action(runDevServerCmd)
 
 prog.parse(process.argv)

packages/api/scripts/cmds/db-types.js

@@ -1,18 +1,14 @@
 import path from 'path'
 import { fileURLToPath } from 'node:url'
 import execa from 'execa'
-import { servicesStartCmd, servicesStopCmd } from './services.js'
+import { runWithServices } from './services.js'
 import delay from 'delay'
 import { dbSqlCmd } from './db-sql.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
 export async function dbTypesCmd() {
-  const project = `nft-storage-db-types-${Date.now()}`
-  await servicesStartCmd({ project })
-  await delay(2000)
-
-  try {
+  await runWithServices(async () => {
     await dbSqlCmd({ cargo: true, testing: true })
     await delay(2000)
     const url = `${process.env.DATABASE_URL}/?apikey=${process.env.DATABASE_TOKEN}`
@@ -29,7 +25,5 @@ export async function dbTypesCmd() {
         preferLocal: true,
       }
     )
-  } finally {
-    await servicesStopCmd({ clean: true, project })
-  }
+  })
 }

packages/api/scripts/cmds/minio.js

@@ -2,29 +2,32 @@ import { Client as Minio } from 'minio'
 import retry from 'p-retry'
 import { isPortReachable } from '../utils.js'
 
-export const MINIO_API_PORT = process.env.MINIO_API_PORT
-  ? Number.parseInt(process.env.MINIO_API_PORT)
-  : 9000
-
-const minioConfig = {
-  useSSL: false,
-  endPoint: '127.0.0.1',
-  port: MINIO_API_PORT,
-  accessKey: 'minioadmin',
-  secretKey: 'minioadmin',
+function minioConfig() {
+  const port = process.env.MINIO_API_PORT
+    ? Number.parseInt(process.env.MINIO_API_PORT)
+    : 9000
+
+  return {
+    useSSL: false,
+    endPoint: '127.0.0.1',
+    port: port,
+    accessKey: 'minioadmin',
+    secretKey: 'minioadmin',
+  }
 }
 
 /**
  * @param {string} name Bucket name
  */
 export async function minioBucketCreateCmd(name) {
+  const config = minioConfig()
   await retry(async () => {
-    if (!(await isPortReachable(MINIO_API_PORT))) {
-      throw new Error(`Minio API not reachable on port: ${MINIO_API_PORT}`)
+    if (!(await isPortReachable(config.port))) {
+      throw new Error(`Minio API not reachable on port: ${config.port}`)
     }
   })
 
-  const minio = new Minio(minioConfig)
+  const minio = new Minio(config)
 
   if (await minio.bucketExists(name)) {
     return console.log(`Cannot create bucket "${name}": already exists`)
@@ -38,7 +41,7 @@ export async function minioBucketCreateCmd(name) {
  * @param {string} name Bucket name
  */
 export async function minioBucketRemoveCmd(name) {
-  const minio = new Minio(minioConfig)
+  const minio = new Minio(minioConfig())
 
   if (!(await minio.bucketExists(name))) {
     return console.log(`Cannot remove bucket "${name}": not found`)

packages/api/scripts/cmds/run-dev.js

@@ -0,0 +1,44 @@
+import execa from 'execa'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+import { runWithServices } from './services.js'
+import { dbSqlCmd } from './db-sql.js'
+import { minioBucketCreateCmd } from './minio.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const workerPath = path.join(__dirname, '../../dist/worker.js')
+const envFilePath = path.join(__dirname, '../../../../.env')
+
+/**
+ *
+ * @param {object} opts
+ * @param {boolean} opts.services
+ * @param {boolean} opts.persistent
+ */
+export async function runDevServerCmd({ persistent, services }) {
+  const action = async () => {
+    console.log('initializing DB schema')
+    await dbSqlCmd({ cargo: true, testing: true, reset: true })
+
+    console.log('creating minio bucket')
+    await minioBucketCreateCmd('dotstorage-dev-0')
+
+    console.log('running dev server')
+    await runMiniflare(workerPath, '--watch', '--debug', '--env', envFilePath)
+  }
+
+  if (services) {
+    await runWithServices(action, { persistent })
+  } else {
+    await action()
+  }
+}
+
+/**
+ *
+ * @param {string[]} miniflareArgs
+ */
+async function runMiniflare(...miniflareArgs) {
+  await execa('npx', ['miniflare', ...miniflareArgs], { stdio: 'inherit' })
+}