api_refs[minor]: Typedoc monorepo support (langchain-ai#6458)

* api_refs[minor]: Typedoc monorepo support * fixed da build yo * added redirects * fix homepage readme * cr * 50% concurrency * 1 concurrency * no cache * cr * cr * cr * cr * build more in vercel * deplpy w vercel * use api ref docs project * cr * build examples as its own step in gh action * install deps first * cr * add prod deployment too * cr * cr * try another redirect * more redirects * try another * fix redirects * added deploy docs prod action * dont deploy docs via gh action, only api refs * cr * cr * cr
tyllenb · Aug 12, 2024 · 4645f29 · 4645f29
1 parent 0551b1c
commit 4645f29
Show file tree

Hide file tree

Showing 14 changed files with 459 additions and 217 deletions.
diff --git a/.github/workflows/deploy-api-refs-preview.yml b/.github/workflows/deploy-api-refs-preview.yml
@@ -0,0 +1,43 @@
+name: Deploy API Refs Preview
+
+on:
+  workflow_dispatch:  # Allows triggering the workflow manually in GitHub UI
+  pull_request:
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: exports-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy-api-refs-preview:
+    name: Deploy Preview API Refs to Vercel
+    runs-on: ubuntu-latest
+    environment: Staging
+    env:
+      VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
+      VERCEL_PROJECT_ID: ${{ secrets.VERCEL_API_DOCS_PROJECT_ID }}
+      VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+      PR_URL: ${{ github.event.pull_request.html_url }}
+      GITHUB_TOKEN: ${{ secrets.PR_COMMENT_GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Vercel CLI
+        run: npm install --global vercel@latest
+      - name: Pull Vercel Environment Information
+        run: vercel pull --yes --environment=preview --token=${{ secrets.VERCEL_TOKEN }}
+      - name: Install dependencies
+        run: yarn install --immutable --mode=skip-build
+      - name: Build All Projects
+        run: yarn turbo:command build --filter=!examples --filter=!api_refs --filter=!core_docs --filter=!create-langchain-integration
+      - name: Build Project Artifacts
+        run: vercel build --token=${{ secrets.VERCEL_TOKEN }}
+      - name: Deploy Project Artifacts to Vercel
+        run: vercel deploy --prebuilt --token=${{ secrets.VERCEL_TOKEN }} > deployment-url-api-refs.txt
+      - name: Post Preview URL Comment
+        run: gh pr comment $PR_URL -F deployment-url-api-refs.txt
diff --git a/.github/workflows/deploy-api-refs-prod.yml b/.github/workflows/deploy-api-refs-prod.yml
@@ -0,0 +1,39 @@
+name: Deploy API Refs Prod
+
+on:
+  workflow_dispatch:  # Allows triggering the workflow manually in GitHub UI
+  push:
+    branches: ["main"]
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: exports-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy-api-refs-prod:
+    name: Deploy Prod API Refs to Vercel
+    runs-on: ubuntu-latest
+    environment: Production
+    env:
+      VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
+      VERCEL_PROJECT_ID: ${{ secrets.VERCEL_API_DOCS_PROJECT_ID }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Vercel CLI
+        run: npm install --global vercel@latest
+      - name: Pull Vercel Environment Information
+        run: vercel pull --yes --environment=production --token=${{ secrets.VERCEL_TOKEN }}
+      - name: Install dependencies
+        run: yarn install --immutable --mode=skip-build
+      - name: Build All Projects
+        run: yarn turbo:command build --filter=!examples --filter=!api_refs --filter=!core_docs --filter=!create-langchain-integration
+      - name: Build Project Artifacts
+        run: vercel build --prod --token=${{ secrets.VERCEL_TOKEN }}
+      - name: Deploy Project Artifacts to Vercel
+        run: vercel deploy --prebuilt --prod --token=${{ secrets.VERCEL_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -46,3 +46,5 @@ docs/api_refs/typedoc.json
 
 .tool-versions
 credentials.json
+**/typedoc.json
+.vercel
diff --git a/docs/api_refs/package.json b/docs/api_refs/package.json
@@ -7,7 +7,6 @@
     "typedoc:build": "npx typedoc --options typedoc.json",
     "build:scripts": "node ./scripts/create-entrypoints.js && yarn typedoc:build && node ./scripts/update-typedoc-css.js",
     "build": "yarn clean && yarn build:scripts && next build",
-    "build:vercel": "yarn clean && yarn build:scripts && next build",
     "start": "yarn build && next start -p 3001",
     "lint": "next lint",
     "format": "prettier --write \"**/*.{js,jsx,ts,tsx}\"",
@@ -26,10 +25,12 @@
     "autoprefixer": "^10.0.1",
     "eslint": "^8",
     "eslint-config-next": "14.0.1",
+    "glob": "^10.3.10",
     "postcss": "^8",
     "prettier": "^2.8.3",
     "tailwindcss": "^3.3.0",
-    "typedoc": "^0.26.0",
+    "ts-morph": "^23.0.0",
+    "typedoc": "^0.26.1",
     "typedoc-plugin-expand-object-like-types": "^0.1.2",
     "typescript": "~5.1.6"
   }

diff --git a/docs/api_refs/scripts/create-entrypoints.js b/docs/api_refs/scripts/create-entrypoints.js
@@ -1,5 +1,6 @@
 const fs = require("fs");
 const path = require("path");
+const { execSync } = require("child_process");
 
 const BASE_TYPEDOC_CONFIG = {
   $schema: "https://typedoc.org/schema.json",
@@ -12,11 +13,10 @@ const BASE_TYPEDOC_CONFIG = {
     "alphabetical",
   ],
   plugin: [
-    "./typedoc_plugins/hide_underscore_lc.js",
+    "./scripts/typedoc-plugin.js",
     "typedoc-plugin-expand-object-like-types",
   ],
   tsconfig: "../../tsconfig.json",
-  readme: "none",
   excludePrivate: true,
   excludeInternal: true,
   excludeExternals: false,
@@ -29,6 +29,7 @@ const BASE_TYPEDOC_CONFIG = {
   skipErrorChecking: true,
   exclude: ["dist"],
   hostedBaseUrl: "https://v02.api.js.langchain.com/",
+  entryPointStrategy: "packages",
 };
 
 /**
@@ -42,6 +43,35 @@ const updateJsonFile = (relativePath, updateFunction) => {
   fs.writeFileSync(relativePath, JSON.stringify(res, null, 2) + "\n");
 };
 
+const workspacesListBreakStr = `"}
+{"`;
+const workspacesListJoinStr = `"},{"`;
+const BLACKLISTED_WORKSPACES = [
+  "@langchain/azure-openai",
+  "@langchain/google-gauth",
+  "@langchain/google-webauth",
+];
+
+/**
+ * @returns {Array<string>} An array of paths to all workspaces in the monorepo.
+ */
+function getYarnWorkspaces() {
+  const stdout = execSync("yarn workspaces list --json");
+  const workspaces = JSON.parse(
+    `[${stdout
+      .toString()
+      .split(workspacesListBreakStr)
+      .join(workspacesListJoinStr)}]`
+  );
+  const cleanedWorkspaces = workspaces.filter(
+    (ws) =>
+      ws.name === "langchain" ||
+      (ws.name.startsWith("@langchain/") &&
+        !BLACKLISTED_WORKSPACES.find((blacklisted) => ws.name === blacklisted))
+  );
+  return cleanedWorkspaces.map((ws) => `../../${ws.location}`);
+}
+
 async function main() {
   const workspaces = fs
     .readdirSync("../../libs/")
@@ -61,8 +91,6 @@ async function main() {
     fs.readFileSync("./blacklisted-entrypoints.json")
   );
 
-  const entrypoints = new Set([]);
-
   for await (const configFile of configFiles) {
     const langChainConfig = await import(configFile);
     if (!("entrypoints" in langChainConfig.config)) {
@@ -73,7 +101,7 @@ async function main() {
       langChainConfig.config.entrypoints === null ||
       langChainConfig.config.entrypoints === undefined
     ) {
-      return;
+      continue;
     }
     const { config } = langChainConfig;
 
@@ -85,7 +113,7 @@ async function main() {
     const deprecatedNodeOnly =
       "deprecatedNodeOnly" in config ? config.deprecatedNodeOnly : [];
 
-    Object.values(config.entrypoints)
+    const workspaceEntrypoints = Object.values(config.entrypoints)
       .filter((key) => !deprecatedNodeOnly.includes(key))
       .filter(
         (key) =>
@@ -94,16 +122,43 @@ async function main() {
               blacklistedItem === `${entrypointDir}/src/${key}.ts`
           )
       )
-      .map((key) => entrypoints.add(`${entrypointDir}/src/${key}.ts`));
+      .map((key) => `src/${key}.ts`);
+
+    const typedocPath = path.join(entrypointDir, "typedoc.json");
+
+    if (!fs.existsSync(typedocPath)) {
+      fs.writeFileSync(typedocPath, "{}\n");
+    }
+
+    updateJsonFile(typedocPath, (existingConfig) => ({
+      ...existingConfig,
+      entryPoints: workspaceEntrypoints,
+      extends: typedocPath.includes("/libs/")
+        ? ["../../docs/api_refs/typedoc.base.json"]
+        : ["../docs/api_refs/typedoc.base.json"],
+    }));
   }
+
   // Check if the `./typedoc.json` file exists, since it is gitignored by default
   if (!fs.existsSync("./typedoc.json")) {
     fs.writeFileSync("./typedoc.json", "{}\n");
   }
 
+  const yarnWorkspaces = getYarnWorkspaces();
+
   updateJsonFile("./typedoc.json", () => ({
     ...BASE_TYPEDOC_CONFIG,
-    entryPoints: Array.from(entrypoints),
+    entryPoints: yarnWorkspaces,
   }));
 }
-main();
+
+async function runMain() {
+  try {
+    await main();
+  } catch (error) {
+    console.error("An error occurred while creating the entrypoints.");
+    throw error;
+  }
+}
+
+runMain();