Monitor all fetch requests (#5)

* Prototyping monkeypatched fetch

* Remove todo

* Import replaceFetch

* Add my gitignore

* Replace neon with libsql

* Fix json parsing of messages and add support for drizzle studio

* Define db in middleware for the hono app

* Remove the node:dev script

* Add comment to the db middleware

* Remove wrangler toml from api to reduce confusion

* Barebones prototype of npx support

* Improve migration logic for when we are in dist folder

* Hacky ability to serve frontend

* Add catchall for frontend routes

* Allow configuring the port for mizu via env var MIZU_PORT

* Start preparing a publish

* Remove @/ imports because they did not get transpiled correctly and NOW IT WORKS kinda

* Proxy api requests in vite on frontend to the api server

* Update readme and package.json

* Suppress type errors in find-source-function.ts

* Update tsconfig and then appease typescript

- changed types to "node" instead of cloudflare workers
- changed "module" to NodeNext, which required renaming a bunch of imports

* Remove more @/ imports because they do not play nicely with node

* Add comments to some confusing helpers

* Modify additional files after transition to node modules

* Bump package.json

* Fix more conflicts and format

* Modify biome to ignore all files in dist folders (including d.ts, which caused issues for me)

* Organize import test

* Okay vscode is confused about import org

* Change cli.cjs to cli.js

* Hackily automigrate

* Fix cli.js to work with modern node

* Simplify build and update readmes

* Update root readme

* Format

* Update api package.json license

* Ignore the biome config from published package

* Update readme

* Improve comments in cli.js

* Add comments about __dirname shim

* Write a comment to explain the getMigrationsFolder function

* Move typescript to dev deps

* Remove wrangler from the api

* Ignore drizzle.config.ts in the published package

* Bump package version of api to 0.0.8

* Format

* Update comment in index.node.ts about the GET catchall

* Fix references to renamed scripts folder (now test-content)

* Update npmignore and package.json in api after merge

* Format

* Prepare client library for publishing on npm

* Remove postinstall script from client-library

* Update README

* Add default createConfig to createHonoMiddleware

* Make installation easier

* Rename to @mizu-dev/hono

* Update package-lock

* Fill in some fetch request details

* Put res.clone() in a try-catch to avoid throwing errors for ws responses (like what gerry encountered)

* Format

* Prep beta version of client library with fetch support

* Add response headers

* Try adding a friendly link when responses are logged

* Update README

* Factor out utilities

* Format

* Implement configuration to turn off fetch monkeypatching

* Update readme with descriptions of monitor config opetions

* Update replaceFetch to send headers

* Update comments relating to the "ignored logs"

* Fix requestId, add end and elapsed to fetch_error, and factor out logic for obtainign req and res data

* Format

* Add some better frontend support for new fetch lifecycle events

* Format

* Add zod schema for fetch args

* Format

* Update development instructions and ignore packed files from git + npm published pkg

* Update development.md

* Update package version before publish

* Update base package.json description

* Add frontend support for fetch_logging_error

* Add support for fetch error messages in ui and implement fetch response unknown (middleware errored when parsing response)

* Bump mizu-studio version

* Add TODO about not failing silently in tryPrettyPrintLoggerLog

* Rename heinously long name of util function in api

api/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.8",
+  "version": "0.0.9",
   "name": "@mizu-dev/studio",
   "description": "Local development debugging interface for Hono apps",
   "author": "Fiberplane<[email protected]>",

client-library/.gitignore

@@ -12,3 +12,6 @@ bun.lockb
 
 # mac
 .DS_Store
+
+# packed files
+mizu-dev-hono-*.tgz

client-library/.npmignore

@@ -2,6 +2,9 @@ node_modules
 .dev.vars
 .dev.vars.example
 
+# do not publish packed files (from local testing)
+mizu-dev-hono-*.tgz
+
 # do not publish src
 src/*
 

client-library/DEVELOPMENT.md

@@ -0,0 +1,43 @@
+# Development
+
+To test the middleware package locally, without publishing it on npm, you can use either `npm link` or `npm pack`.
+
+## `npm link`
+
+Build and link the package locally (from the `client-library` folder)
+
+```sh
+# In the client-library folder
+npm run build
+npm link
+
+# Change directories to another project that uses mizu hono middleware
+cd /path/to/test/project
+npm link @mizu-dev/hono
+```
+
+No need to install the package in your other project, just import it:
+
+```ts
+import { createHonoMiddleware } from "@mizu-dev/hono";
+```
+
+Then, once you are done testing locally, remember to unlink the package
+
+```sh
+# check the link exists
+npm ls -g --depth=0 --link=true
+# remove the link
+npm unlink @mizu-dev/hono -g
+```
+
+## `npm pack`
+
+This command will create a tarball that you can install as a file reference in another project.
+
+```sh
+npm pack
+cd /path/to/other/projenct
+# Replace the version command with the tarball that was created
+npm install /path/to/client-library/@mizu-dev-hono-x.y.z.tgz
+```

client-library/README.md

@@ -87,9 +87,9 @@ const createConfig = (c: Context) => {
   service: c.env?.SERVICE_NAME || "unknown",
   libraryDebugMode: c.env?.LIBRARY_DEBUG_MODE,
   monitor: {
-   fetch: true,
-   logging: true,
-   requests: true,
+   fetch: true, // set to false if you do not want to monkey-path fetch and send data about external network requests to mizu
+   logging: true, // not yet implemented!
+   requests: true, // set to false if you do not want to log data about each request and response to mizu
   },
  };
 }
@@ -120,4 +120,8 @@ Make requests to your Hono app, and the logs should show up in the mizu UI!
 npx @mizu-dev/studio
 ```
 
-That's it! You should see your logs in the mizu UI.
+That's it! You should see your logs in the mizu UI.
+
+## Local Development
+
+See [DEVELOPMENT.md](./DEVELOPMENT.md) for instructions on how to develop this library.

client-library/package.json

@@ -4,7 +4,7 @@
   "author": "Fiberplane<[email protected]>",
   "type": "module",
   "main": "dist/index.js",
-  "version": "0.0.1",
+  "version": "0.1.0-beta.3",
   "dependencies": {
     "@neondatabase/serverless": "^0.9.3",
     "hono": "^4.3.9"

client-library/src/honoMiddleware.ts

@@ -1,13 +1,16 @@
 import type { NeonDbError } from "@neondatabase/serverless";
 import type { Context } from "hono";
+import { replaceFetch } from "./replace-fetch";
 import { RECORDED_CONSOLE_METHODS, log } from "./request-logger";
 import {
   errorToJson,
   extractCallerLocation,
   generateUUID,
   neonDbErrorToJson,
   polyfillWaitUntil,
+  shouldIgnoreMizuLog,
   shouldPrettifyMizuLog,
+  tryCreateFriendlyLink,
   tryPrettyPrintLoggerLog,
 } from "./utils";
 
@@ -18,10 +21,11 @@ type Config = {
   /** Use `libraryDebugMode` to log into the terminal what we are sending to the Mizu server on each request/response */
   libraryDebugMode?: boolean;
   monitor: {
-    // TODO - implement this control/feature
+    /** Send data to mizu about each fetch call made during a handler's lifetime */
     fetch: boolean;
     // TODO - implement this control/feature
     logging: boolean;
+    /** Send data to mizu about each incoming request and outgoing response */
     requests: boolean;
   };
 };
@@ -54,7 +58,7 @@ export function createHonoMiddleware(options?: {
         fetch: monitorFetch,
         // TODO - implement these controls/features
         // logging: monitorLogging,
-        // requests: monitorRequests,
+        requests: monitorRequests,
       },
     } = createConfig(c);
     const ctx = c.executionCtx;
@@ -65,6 +69,13 @@ export function createHonoMiddleware(options?: {
 
     const teardownFunctions: Array<() => void> = [];
 
+    const { originalFetch, undo: undoReplaceFetch } = replaceFetch({
+      skipMonkeyPatch: !monitorFetch,
+    });
+    // We need to undo our monkeypatching since workers can operate in a shared environment
+    // This is similar to how we need to undo our monkeypatching of `console.*` methods (see HACK comment below)
+    teardownFunctions.push(undoReplaceFetch);
+
     // TODO - (future) Take the traceId from headers but then fall back to uuid here?
     const traceId = generateUUID();
 
@@ -106,25 +117,50 @@ export function createHonoMiddleware(options?: {
           timestamp,
         };
         ctx.waitUntil(
-          fetch(endpoint, {
+          // Use `originalFetch` to avoid an infinite loop of logging to mizu
+          // If we use our monkeyPatched version, then each fetch logs to mizu,
+          // which triggers another fetch to log to mizu, etc.
+          originalFetch(endpoint, {
             method: "POST",
             headers: {
               "Content-Type": "application/json",
             },
             body: JSON.stringify(payload),
           }),
         );
+
         const applyArgs = args?.length ? [message, ...args] : [message];
+
+        // To explain the use of this `shouldIgnoreMizuLog` function a bit more:
+        //
+        // The middleware itself uses `console.log` and `console.error` to send logs to mizu.
+        //
+        // Specifically, it does this in the monkeypatched version of `fetch`.
+        //
+        // So, we want to short circuit those logs and not actually print them to the user's console
+        // Otherwise, things get realllyyyy noisy.
+        //
+        if (shouldIgnoreMizuLog(applyArgs)) {
+          return;
+        }
+
         if (!libraryDebugMode && shouldPrettifyMizuLog(applyArgs)) {
-          // HACK - Try parsing the message as json and extracting all the fields we care about logging prettily
-          tryPrettyPrintLoggerLog(originalConsoleMethod, message);
+          // Optionally log a link to the mizu dashboard for the "response" log
+          const linkToMizuUi = tryCreateFriendlyLink({
+            message,
+            traceId,
+            mizuEndpoint: endpoint,
+          });
+
+          // Try parsing the message as json and extracting all the fields we care about logging prettily
+          tryPrettyPrintLoggerLog(originalConsoleMethod, message, linkToMizuUi);
         } else {
           originalConsoleMethod.apply(originalConsoleMethod, applyArgs);
         }
       };
     }
 
-    if (monitorFetch) {
+    if (monitorRequests) {
       await log(c, next);
     } else {
       await next();

client-library/src/replace-fetch.ts

@@ -0,0 +1,185 @@
+import { IGNORE_MIZU_LOGGER_LOG, errorToJson, generateUUID } from "./utils";
+
+/**
+ * Hacky function that monkey-patches fetch to send data about network requests to mizu.
+ *
+ * We log data with the `IGNORE_MIZU_LOGGER_LOG` symbol in order to avoid printing it to the user's console.
+ * That symbol is used by the monkey-patched console.* methods to avoid _actually_ printing the logs to the console.
+ * Seem confusing? Yes. Yes it is. However, relying on our monkey-patched console.* methods allows us to make use of the current traceId, etc.
+ *
+ * This function also has an option to skip monkey-patching, so that the user can configure whether or not to use this functionality.
+ *
+ * Returns the original fetch (the fetch we're monkey patching), so the middleware can still use it to send log data to mizu.
+ * Returns an `undo` function, so that the middleware can undo the monkey-patching after the request is finished.
+ * (This "undo" functionality is really important in cloudflare workers!)
+ *
+ * In future, when writing an otel-compatible version of this, we should take inspo from:
+ *
+ * https://github.com/evanderkoogh/otel-cf-workers/blob/6f1c79056776024fd3e816b9e3991527e7217510/src/instrumentation/fetch.ts#L198
+ */
+export function replaceFetch({
+  skipMonkeyPatch,
+}: { skipMonkeyPatch: boolean }) {
+  const originalFetch = globalThis.fetch;
+
+  if (skipMonkeyPatch) {
+    return { originalFetch, undo: () => {} };
+  }
+
+  // @ts-ignore
+  globalThis.fetch = async (...args) => {
+    const requestId = generateUUID();
+    const start = Date.now();
+    const {
+      url,
+      method,
+      body,
+      headers: requestHeaders,
+    } = await getRequestData(...args);
+
+    console.log(
+      JSON.stringify({
+        lifecycle: "fetch_start",
+        requestId,
+        start,
+        url, // Parsed URL
+        method, // Parsed method
+        body, // Parsed body
+        headers: requestHeaders, // Parsed headers
+        args, // Full request args
+      }),
+      IGNORE_MIZU_LOGGER_LOG,
+    );
+
+    try {
+      const response = await originalFetch(...args);
+      const end = Date.now();
+      const elapsed = end - start;
+
+      const clonedResponse = response.clone();
+
+      if (!clonedResponse.ok) {
+        const { body, headers, status, statusText } =
+          // @ts-ignore: weird type conflict between cloned response and `Response` type
+          await getResponseData(clonedResponse);
+
+        // Count any not-ok responses as a fetch_error
+        console.error(
+          JSON.stringify({
+            lifecycle: "fetch_error",
+            requestId,
+            status,
+            statusText,
+            body,
+            url,
+            headers,
+            end,
+            elapsed,
+          }),
+          IGNORE_MIZU_LOGGER_LOG,
+        );
+      }
+
+      const { body, headers, status, statusText } =
+        // @ts-ignore: weird type conflict between cloned response and `Response` type
+        await getResponseData(clonedResponse);
+
+      console.log(
+        JSON.stringify({
+          lifecycle: "fetch_end",
+          requestId,
+          end,
+          elapsed,
+          url,
+          status,
+          statusText,
+          headers,
+          body,
+        }),
+        IGNORE_MIZU_LOGGER_LOG,
+      );
+
+      return response;
+    } catch (err) {
+      console.error(
+        JSON.stringify({
+          lifecycle: "fetch_logging_error",
+          requestId,
+          url,
+          error: err instanceof Error ? errorToJson(err) : err,
+        }),
+        IGNORE_MIZU_LOGGER_LOG,
+      );
+      throw err;
+    }
+  };
+
+  return {
+    undo: () => {
+      globalThis.fetch = originalFetch;
+    },
+    originalFetch,
+  };
+}
+
+async function tryGetResponseBodyAsText(response: Response) {
+  try {
+    return await response.text();
+  } catch {
+    return null;
+  }
+}
+
+function getResponseHeaders(clonedResponse: Response) {
+  // Extract and format response headers
+  const headers: { [key: string]: string } = {};
+  clonedResponse.headers.forEach((value, key) => {
+    headers[key] = value;
+  });
+  return headers;
+}
+
+async function getRequestData(...args: Parameters<typeof fetch>) {
+  const [resource, init] = args;
+  const method = init?.method || "GET";
+
+  const url =
+    typeof resource === "string"
+      ? resource
+      : resource instanceof URL
+        ? resource.toString()
+        : resource.url;
+
+  const body = init?.body ? await new Response(init.body).text() : null;
+
+  const requestHeaders: { [key: string]: string } = {};
+  if (init?.headers) {
+    const headers = new Headers(init.headers);
+    headers.forEach((value, key) => {
+      requestHeaders[key] = value;
+    });
+  }
+
+  return {
+    url,
+    method,
+    body,
+    headers: requestHeaders,
+  };
+}
+
+async function getResponseData(clonedResponse: Response) {
+  const body: string | null =
+    // @ts-ignore: weird type conflict between cloned response and `Response` type
+    await tryGetResponseBodyAsText(clonedResponse);
+
+  // @ts-ignore: weird type conflict between cloned response and `Response` type
+  const headers = getResponseHeaders(clonedResponse);
+
+  return {
+    body,
+    headers,
+    status: clonedResponse.status,
+    statusText: clonedResponse.statusText,
+  };
+}