Merge pull request #1993 from continuedev/pe/clear-index

feat: create "rebuild index" dialog and command

core/core.ts

@@ -638,8 +638,13 @@ export class Core {
       const rows = await DevDataSqliteDb.getTokensPerModel();
       return rows;
     });
-    on("index/forceReIndex", async (msg) => {
-      const dirs = msg.data ? [msg.data] : await this.ide.getWorkspaceDirs();
+    on("index/forceReIndex", async ({ data }) => {
+      if (data?.shouldClearIndexes) {
+        const codebaseIndexer = await this.codebaseIndexerPromise;
+        await codebaseIndexer.clearIndexes();
+      }
+
+      const dirs = data?.dir ? [data.dir] : await this.ide.getWorkspaceDirs();
       await this.refreshCodebaseIndex(dirs);
     });
     on("index/setPaused", (msg) => {

core/index.d.ts

@@ -37,6 +37,7 @@ export interface Chunk extends ChunkWithoutID {
 export interface IndexingProgressUpdate {
   progress: number;
   desc: string;
+  shouldClearIndexes?: boolean;
   status: "loading" | "indexing" | "done" | "failed" | "paused" | "disabled";
 }
 

core/indexing/CodebaseIndexer.ts

@@ -1,6 +1,7 @@
 import { ConfigHandler } from "../config/ConfigHandler.js";
 import { IContinueServerClient } from "../continueServer/interface.js";
 import { IDE, IndexTag, IndexingProgressUpdate } from "../index.js";
+import { getIndexSqlitePath, getLanceDbPath } from "../util/paths.js";
 import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex.js";
 import { CodeSnippetsCodebaseIndex } from "./CodeSnippetsIndex.js";
 import { FullTextSearchCodebaseIndex } from "./FullTextSearch.js";
@@ -12,6 +13,7 @@ import {
   RefreshIndexResults,
 } from "./types.js";
 import { walkDirAsync } from "./walkDir.js";
+import * as fs from "fs/promises";
 
 export class PauseToken {
   constructor(private _paused: boolean) {}
@@ -26,13 +28,41 @@ export class PauseToken {
 }
 
 export class CodebaseIndexer {
+  // Note that we exclude certain Sqlite errors that we do not want to clear the indexes on,
+  // e.g. a `SQLITE_BUSY` error.
+  errorsRegexesToClearIndexesOn = [
+    /Invalid argument error: Values length (d+) is less than the length ((d+)) multiplied by the value size (d+)/,
+    /SQLITE_CONSTRAINT/,
+    /SQLITE_ERROR/,
+    /SQLITE_CORRUPT/,
+    /SQLITE_IOERR/,
+    /SQLITE_FULL/,
+  ];
+
   constructor(
     private readonly configHandler: ConfigHandler,
     protected readonly ide: IDE,
     private readonly pauseToken: PauseToken,
     private readonly continueServerClient: IContinueServerClient,
   ) {}
 
+  async clearIndexes() {
+    const sqliteFilepath = getIndexSqlitePath();
+    const lanceDbFolder = getLanceDbPath();
+
+    try {
+      await fs.unlink(sqliteFilepath);
+    } catch (error) {
+      console.error(`Error deleting ${sqliteFilepath} folder: ${error}`);
+    }
+
+    try {
+      await fs.rm(lanceDbFolder, { recursive: true, force: true });
+    } catch (error) {
+      console.error(`Error deleting ${lanceDbFolder}: ${error}`);
+    }
+  }
+
   protected async getIndexesToBuild(): Promise<CodebaseIndex[]> {
     const config = await this.configHandler.loadConfig();
     const pathSep = await this.ide.pathSep();
@@ -203,7 +233,6 @@ export class CodebaseIndexer {
         yield this.handleErrorAndGetProgressUpdate(err);
         return;
       }
-      completedDirs += 1;
     }
     yield {
       progress: 100,
@@ -227,20 +256,25 @@ export class CodebaseIndexer {
   }
 
   private errorToProgressUpdate(err: Error): IndexingProgressUpdate {
-    const errorRegex =
-      /Invalid argument error: Values length (\d+) is less than the length \((\d+)\) multiplied by the value size \(\d+\)/;
-    const match = err.message.match(errorRegex);
-    let errMsg: string;
-    if (match) {
-      const [_, valuesLength, expectedLength] = match;
-      errMsg = `Generated embedding had length ${valuesLength} but was expected to be ${expectedLength}. This may be solved by deleting ~/.continue/index and refreshing the window to re-index.`;
-    } else {
-      errMsg = `${err}`;
+    let errMsg: string = `${err}`;
+    let shouldClearIndexes = false;
+
+    // Check if any of the error regexes match
+    for (const regexStr of this.errorsRegexesToClearIndexesOn) {
+      const regex = new RegExp(regexStr);
+      const match = err.message.match(regex);
+
+      if (match !== null) {
+        shouldClearIndexes = true;
+        break;
+      }
     }
+
     return {
       progress: 0,
       desc: errMsg,
       status: "failed",
+      shouldClearIndexes,
     };
   }
 

core/protocol/core.ts

@@ -130,7 +130,10 @@ export type ToCoreFromIdeOrWebviewProtocol = {
     { model: string; promptTokens: number; generatedTokens: number }[],
   ];
   "index/setPaused": [boolean, void];
-  "index/forceReIndex": [undefined | string, void];
+  "index/forceReIndex": [
+    undefined | { dir?: string; shouldClearIndexes?: boolean },
+    void,
+  ];
   "index/indexingProgressBarInitialized": [undefined, void];
   completeOnboarding: [
     {

docs/docs/troubleshooting.md

@@ -49,7 +49,7 @@ On JetBrains, the "pre-release" happens through their Early Access Program (EAP)
 1. Open JetBrains settings (cmd/ctrl+,) and go to "Plugins"
 2. Click the gear icon at the top
 3. Select "Manage Plugin Repositories..."
-4. Add "https://plugins.jetbrains.com/plugins/eap/list" to the list
+4. Add "<https://plugins.jetbrains.com/plugins/eap/list>" to the list
 5. You'll now always be able to download the latest EAP version from the marketplace
 
 ## Download an Older Version
@@ -108,6 +108,12 @@ By default the Continue window is on the left side of VS Code, but it can be dra
 
 If you have entered a valid API key and model, but are still getting a 404 error from OpenAI, this may be because you need to add credits to your billing account. You can do so from the [billing console](https://platform.openai.com/settings/organization/billing/overview). If you just want to check that this is in fact the cause of the error, you can try adding $1 to your account and checking whether the error persists.
 
+### Indexing issues
+
+If you are having persistent errors with indexing, our recommendation is to rebuild your index from scratch. Note that for large codebases this may take some time.
+
+This can be accomplished using the following command: `Continue: Rebuild codebase index`.
+
 ## Still having trouble?
 
 Create a GitHub issue [here](https://github.com/continuedev/continue/issues/new?assignees=&labels=bug&projects=&template=bug-report-%F0%9F%90%9B.md&title=), leaving the details of your problem, and we'll be able to more quickly help you out.

extensions/vscode/package.json

@@ -252,6 +252,12 @@
         "title": "Codebase Force Re-Index",
         "group": "Continue"
       },
+      {
+        "command": "continue.rebuildCodebaseIndex",
+        "category": "Continue",
+        "title": "Rebuild codebase index",
+        "group": "Continue"
+      },
       {
         "command": "continue.docsIndex",
         "category": "Continue",
@@ -276,11 +282,6 @@
         "mac": "cmd+shift+l",
         "key": "ctrl+shift+l"
       },
-      {
-        "command": "continue.toggleAuxiliaryBar",
-        "mac": "alt+cmd+l",
-        "key": "alt+ctrl+l"
-      },
       {
         "command": "continue.acceptDiff",
         "mac": "shift+cmd+enter",

extensions/vscode/src/commands.ts

@@ -284,12 +284,12 @@ const commandsMap: (
 
       streamInlineEdit("docstring", prompt, false, range);
     },
-    "continue.toggleAuxiliaryBar": () => {
-      vscode.commands.executeCommand("workbench.action.toggleAuxiliaryBar");
-    },
     "continue.codebaseForceReIndex": async () => {
       core.invoke("index/forceReIndex", undefined);
     },
+    "continue.rebuildCodebaseIndex": async () => {
+      core.invoke("index/forceReIndex", { shouldClearIndexes: true });
+    },
     "continue.docsIndex": async () => {
       core.invoke("context/indexDocs", { reIndex: false });
     },

extensions/vscode/src/extension/VsCodeExtension.ts

@@ -293,7 +293,7 @@ export class VsCodeExtension {
                   currentBranch !== this.PREVIOUS_BRANCH_FOR_WORKSPACE_DIR[dir]
                 ) {
                   // Trigger refresh of index only in this directory
-                  this.core.invoke("index/forceReIndex", dir);
+                  this.core.invoke("index/forceReIndex", { dir });
                 }
               }
 

gui/src/components/dialogs/ConfirmationDialog.tsx

@@ -36,7 +36,7 @@ function ConfirmationDialog(props: ConfirmationDialogProps) {
   return (
     <div className="p-4">
       <h3>{props.title ?? "Confirmation"}</h3>
-      <p>{props.text}</p>
+      <p style={{ whiteSpace: "pre-wrap" }}>{props.text}</p>
 
       <GridDiv>
         {!!props.hideCancelButton || (