Merge pull request #449 from AikidoSec/beta

Stable release
AikidoSec · Dec 3, 2024 · 4605019 · 4605019
2 parents 27ee22d + 65132df
commit 4605019
Show file tree

Hide file tree

Showing 357 changed files with 76,582 additions and 14,692 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -35,11 +35,8 @@ jobs:
         uses: grafana/setup-k6-action@v1
       - name: Install wrk
         run: |
-          sudo apt-get install build-essential libssl-dev git -y
-          git clone https://github.com/wg/wrk.git wrk
-          cd wrk
-          sudo make
-          sudo cp wrk /usr/local/bin
+          sudo apt-get update
+          sudo apt-get install -y wrk
       - run: make install
       - run: make build
       - name: Run NoSQL Injection Benchmark

diff --git a/.github/workflows/end-to-end-tests.yml b/.github/workflows/end-to-end-tests.yml
@@ -29,6 +29,22 @@ jobs:
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5 -e MYSQL_ROOT_PASSWORD=mypassword -e MYSQL_DATABASE=catsdb --entrypoint sh mysql:8.0 -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"
         ports:
           - "27015:3306"
+      mariadb:
+        image: mariadb:11
+        env:
+          MARIADB_ROOT_PASSWORD: mypassword
+          MARIADB_DATABASE: catsdb
+        ports:
+          - "27018:3306"
+      clickhouse:
+        image: clickhouse/clickhouse-server:24
+        env:
+          "CLICKHOUSE_USER": "clickhouse"
+          "CLICKHOUSE_PASSWORD": "clickhouse"
+          "CLICKHOUSE_DB": "main_db"
+          "CLICKHOUSE_DEFAULT_ACCESS": "MANAGEMENT=1"
+        ports:
+          - "27019:8123"
     timeout-minutes: 10
     strategy:
       matrix:

diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml
@@ -13,4 +13,5 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - run: make install
+      - run: make build
       - run: make lint
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -35,6 +35,22 @@ jobs:
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5 -e MYSQL_ROOT_PASSWORD=mypassword -e MYSQL_DATABASE=catsdb --entrypoint sh mysql:8.0 -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"
         ports:
           - "27015:3306"
+      mariadb:
+        image: mariadb:11
+        env:
+          MARIADB_ROOT_PASSWORD: mypassword
+          MARIADB_DATABASE: catsdb
+        ports:
+          - "27018:3306"
+      clickhouse:
+        image: clickhouse/clickhouse-server:24
+        env:
+          "CLICKHOUSE_USER": "clickhouse"
+          "CLICKHOUSE_PASSWORD": "clickhouse"
+          "CLICKHOUSE_DB": "main_db"
+          "CLICKHOUSE_DEFAULT_ACCESS": "MANAGEMENT=1"
+        ports:
+          - "27019:8123"
     strategy:
       fail-fast: false
       matrix:
@@ -49,6 +65,7 @@ jobs:
         run: |
           sudo echo "127.0.0.1 local.aikido.io" | sudo tee -a /etc/hosts
       - run: make install
+      - run: make build
       - run: make test-ci
       - name: "Upload coverage"
         uses: codecov/[email protected]

diff --git a/Makefile b/Makefile
@@ -1,3 +1,9 @@
+INTERNALS_VERSION = v0.1.31
+INTERNALS_URL = https://github.com/AikidoSec/zen-internals/releases/download/$(INTERNALS_VERSION)
+TARBALL = zen_internals.tgz
+CHECKSUM_FILE = zen_internals.tgz.sha256sum
+INTERNALS_DIR = library/internals
+
 .PHONY: containers
 containers:
 	cd sample-apps && docker-compose up -d --remove-orphans
@@ -62,6 +68,22 @@ lambda-mongodb-safe:
 nestjs-sentry:
 	cd sample-apps/nestjs-sentry && AIKIDO_DEBUG=true AIKIDO_BLOCKING=true NODE_OPTIONS=--preserve-symlinks npm run start
 
+.PHONY: nestjs-fastify
+nestjs-fastify:
+	cd sample-apps/nestjs-fastify && AIKIDO_DEBUG=true AIKIDO_BLOCKING=true NODE_OPTIONS=--preserve-symlinks npm run start
+
+.PHONY: fastify-mysql2
+fastify-mysql2:
+	cd sample-apps/fastify-mysql2 && AIKIDO_DEBUG=true AIKIDO_BLOCKING=true node --preserve-symlinks app.js
+
+.PHONY: koa-sqlite3
+koa-sqlite3:
+	cd sample-apps/koa-sqlite3 && AIKIDO_DEBUG=true AIKIDO_BLOCKING=true node --preserve-symlinks app.js
+
+.PHONY: fastify-clickhouse
+fastify-clickhouse:
+	cd sample-apps/fastify-clickhouse && AIKIDO_DEBUG=true AIKIDO_BLOCKING=true node app.js
+
 .PHONY: install
 install:
 	mkdir -p build
@@ -74,13 +96,25 @@ install:
 	node scripts/install.js
 
 .PHONY: build
-build:
+build: $(INTERNALS_DIR)/zen_internals.js
 	mkdir -p build
 	rm -r build
 	cd library && npm run build
 	cp README.md build/README.md
 	cp LICENSE build/LICENSE
 	cp library/package.json build/package.json
+	mkdir -p build/internals
+	cp $(INTERNALS_DIR)/zen_internals_bg.wasm build/internals/zen_internals_bg.wasm
+
+$(INTERNALS_DIR)/zen_internals.js: Makefile
+	curl -L $(INTERNALS_URL)/$(TARBALL) -o $(INTERNALS_DIR)/$(TARBALL)
+	curl -L $(INTERNALS_URL)/$(CHECKSUM_FILE) -o $(INTERNALS_DIR)/$(CHECKSUM_FILE)
+	cd $(INTERNALS_DIR) && sha256sum -c $(CHECKSUM_FILE)
+	tar -xzf $(INTERNALS_DIR)/$(TARBALL) -C $(INTERNALS_DIR)
+	touch $@
+	rm $(INTERNALS_DIR)/zen_internals.d.ts
+	rm $(INTERNALS_DIR)/$(TARBALL)
+	rm $(INTERNALS_DIR)/$(CHECKSUM_FILE)
 
 .PHONY: watch
 watch: build

diff --git a/README.md b/README.md
@@ -43,7 +43,9 @@ Zen for Node.js 16+ is compatible with:
 * ✅ [hapi](docs/hapi.md) 21.x
 * ✅ [micro](docs/micro.md) 10.x
 * ✅ [Next.js](docs/next.md) 12.x, 13.x and 14.x
-* ✅ Fastify (upcoming)
+* ✅ [Fastify](docs/fastify.md) 4.x and 5.x
+* ✅ [Koa](docs/koa.md) 2.x
+* ✅ [NestJS](docs/nestjs.md) 10.x
 
 ### Database drivers
 
@@ -52,9 +54,12 @@ Zen for Node.js 16+ is compatible with:
 * ✅ [`pg`](https://www.npmjs.com/package/pg) 8.x and 7.x
 * ✅ [`mysql`](https://www.npmjs.com/package/mysql) 2.x
 * ✅ [`mysql2`](https://www.npmjs.com/package/mysql2) 3.x
+* ✅ [`mariadb`](https://www.npmjs.com/package/mariadb) 3.x
 * ✅ [`sqlite3`](https://www.npmjs.com/package/sqlite3) 5.x
 * ✅ [`node:sqlite`](https://nodejs.org/api/sqlite.html)
 * ✅ [`better-sqlite3`](https://www.npmjs.com/package/better-sqlite3) 11.x, 10.x, 9.x and 8.x
+* ✅ [`postgres`](https://www.npmjs.com/package/postgres) 3.x
+* ✅ [`@clickhouse/client`](https://www.npmjs.com/package/@clickhouse/client) 1.x
 
 ### Cloud providers
 
@@ -87,6 +92,11 @@ See list above for supported database drivers.
 
 * ✅ [`ShellJS`](https://www.npmjs.com/package/shelljs) 0.8.x, 0.7.x
 
+### Routers
+
+* ✅ [`@koa/router`](https://www.npmjs.com/package/@koa/router) 13.x, 12.x, 11.x and 10.x
+
+
 ## Installation
 
 We recommend testing Zen locally or on staging before deploying to production.

diff --git a/benchmarks/api-discovery/benchmark.js b/benchmarks/api-discovery/benchmark.js
@@ -2,20 +2,15 @@
  * Runs benchmarks for the api discovery (api schema collection)
  */
 const { Routes } = require("../../build/agent/Routes");
-const { isFeatureEnabled } = require("../../build/helpers/featureFlags");
 const reqBodies = require("./reqBodies");
+const headers = require("./headers");
+const queryParameters = require("./queryParameters");
+const cookies = require("./cookies");
 
 const MAX_TIME_LIMIT = 0.05; // milliseconds / statement
-const RUN_COUNT = 100;
+const RUN_COUNT = 100000;
 
 function main() {
-  // Enable feature flag
-  process.env.AIKIDO_FEATURE_COLLECT_API_SCHEMA = "true";
-  if (!isFeatureEnabled("COLLECT_API_SCHEMA")) {
-    console.error("Feature COLLECT_API_SCHEMA is not enabled!");
-    process.exit(1);
-  }
-
   const avgTime = getAvgBenchmark();
   if (avgTime > MAX_TIME_LIMIT) {
     console.error(
@@ -29,31 +24,36 @@ function main() {
 
 main();
 
-function getContext(body) {
+function getContext(body, headers, query, cookies) {
   return {
     method: "POST",
     route: "/test",
     headers: {
+      ...headers,
       "content-type": "application/json",
     },
     body,
     remoteAddress: "",
     url: "http://localhost/test",
     routeParams: {},
-    query: {},
-    cookies: {},
+    query: query,
+    cookies: cookies,
     source: "test",
   };
 }
 
-function runBenchmark(routes, body) {
-  const context = getContext(body);
+function runBenchmark(routes, body, headers, query, cookies) {
+  const context = getContext(body, headers, query, cookies);
   const startTime = performance.now();
   routes.addRoute(context);
   const endTime = performance.now();
   return endTime - startTime;
 }
 
+function nextIndex(index, length) {
+  return index === length - 1 ? 0 : index + 1;
+}
+
 /**
  * This function calculates the average time in ms / count of runs
  * @returns average time in milliseconds
@@ -63,13 +63,21 @@ function getAvgBenchmark() {
   let avgTime = 0;
 
   let bodyIndex = 0;
+  let headersIndex = 0;
+  let queryIndex = 0;
+  let cookieIndex = 0;
   for (let i = 0; i < RUN_COUNT; i++) {
-    avgTime += runBenchmark(routes, reqBodies[bodyIndex]);
-    if (bodyIndex === reqBodies.length - 1) {
-      bodyIndex = 0;
-    } else {
-      bodyIndex++;
-    }
+    avgTime += runBenchmark(
+      routes,
+      reqBodies[bodyIndex],
+      headers[headersIndex],
+      queryParameters[queryIndex],
+      cookies[cookieIndex]
+    );
+    bodyIndex = nextIndex(bodyIndex, reqBodies.length);
+    headersIndex = nextIndex(headersIndex, headers.length);
+    queryIndex = nextIndex(queryIndex, queryParameters.length);
+    cookieIndex = nextIndex(cookieIndex, cookies.length);
   }
 
   avgTime = avgTime / RUN_COUNT;

diff --git a/benchmarks/api-discovery/cookies.js b/benchmarks/api-discovery/cookies.js
@@ -0,0 +1,28 @@
+module.exports = [
+  {
+    sessionId: "abc123xyz",
+    userPreferences: "dark-mode=true;language=nl",
+    authToken: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...",
+  },
+  {
+    userId: "user456",
+    cartId: "cart789",
+    theme: "light",
+    recentViewed: "item001,item002,item003",
+  },
+  {
+    auth: "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...",
+    csrfToken: "def456ghi",
+    rememberMe: "true",
+  },
+  {
+    visitCount: "5",
+    lastVisit: "2024-10-28T15:30:00Z",
+    preferences: "text-size=large;color=blue",
+  },
+  {
+    analyticsId: "GA1.2.3456789.1234567890",
+    trackingOptOut: "false",
+    experimentGroup: "B",
+  },
+];
diff --git a/benchmarks/api-discovery/headers.js b/benchmarks/api-discovery/headers.js
@@ -0,0 +1,51 @@
+module.exports = [
+  {
+    accept: "text/fragment+html",
+    "accept-language": "nl,en;q=0.9,en-US;q=0.8",
+    "if-none-match": 'W/"107a08e1219ed6f9293a4b33529190f7"',
+    priority: "u=1, i",
+    "sec-ch-ua":
+      '"Chromium";v="130", "Google Chrome";v="130", "Not?A_Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"macOS"',
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "same-origin",
+    "sec-gpc": "1",
+    "x-requested-with": "XMLHttpRequest",
+  },
+  {
+    accept: "*/*",
+    "accept-language": "nl,en;q=0.9,en-US;q=0.8",
+    "if-none-match": 'W/"10nkg03furb1th6"',
+    "next-router-prefetch": "1",
+    "next-router-state-tree":
+      "%5B%22%22%2C%7B%22children%22%3A%5B%22(dark-theme)%22%2C%7B%22children%22%3A%5B%22(with-footer)%22%2C%7B%22children%22%3A%5B%22(home)%22%2C%7B%22children%22%3A%5B%22__PAGE__%22%2C%7B%7D%2C%22%2F%22%2C%22refresh%22%5D%7D%5D%7D%5D%7D%5D%7D%2Cnull%2Cnull%2Ctrue%5D",
+    "next-url": "/",
+    priority: "u=1, i",
+    rsc: "1",
+    "sec-ch-ua":
+      '"Chromium";v="130", "Google Chrome";v="130", "Not?A_Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"macOS"',
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "same-origin",
+    "sec-gpc": "1",
+    authorization: "Bearer token",
+  },
+  {
+    accept: "application/json, text/javascript, */*; q=0.01",
+    "accept-language": "nl,en;q=0.9,en-US;q=0.8",
+    "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
+    "sec-ch-ua":
+      '"Chromium";v="130", "Google Chrome";v="130", "Not?A_Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"macOS"',
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "same-origin",
+    "x-requested-with": "XMLHttpRequest",
+    token: "token",
+  },
+];