testing: add indexing update tests

core/indexing/CodeSnippetsIndex.ts

@@ -100,26 +100,30 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
     contents: string,
   ): Promise<(ChunkWithoutID & { title: string })[]> {
     const parser = await getParserForFile(filepath);
+
     if (!parser) {
       return [];
     }
+
     const ast = parser.parse(contents);
     const query = await getQueryForFile(filepath, TSQueryType.CodeSnippets);
     const matches = query?.matches(ast.rootNode);
 
-    return (
-      matches?.flatMap((match) => {
-        const node = match.captures[0].node;
-        const title = match.captures[1].node.text;
-        const results = {
-          title,
-          content: node.text,
-          startLine: node.startPosition.row,
-          endLine: node.endPosition.row,
-        };
-        return results;
-      }) ?? []
-    );
+    if (!matches) {
+      return [];
+    }
+
+    return matches.flatMap((match) => {
+      const node = match.captures[0].node;
+      const title = match.captures[1].node.text;
+      const results = {
+        title,
+        content: node.text,
+        startLine: node.startPosition.row,
+        endLine: node.endPosition.row,
+      };
+      return results;
+    });
   }
 
   async *update(
@@ -132,6 +136,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
     await CodeSnippetsCodebaseIndex._createTables(db);
     const tagString = tagToString(tag);
 
+    // Compute
     for (let i = 0; i < results.compute.length; i++) {
       const compute = results.compute[i];
 
@@ -174,6 +179,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
       markComplete([compute], IndexResultType.Compute);
     }
 
+    // Delete
     for (let i = 0; i < results.del.length; i++) {
       const del = results.del[i];
       const deleted = await db.run(
@@ -186,6 +192,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
       markComplete([del], IndexResultType.Delete);
     }
 
+    // Add tag
     for (let i = 0; i < results.addTag.length; i++) {
       const addTag = results.addTag[i];
       let snippets: (ChunkWithoutID & { title: string })[] = [];
@@ -220,6 +227,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
       markComplete([results.addTag[i]], IndexResultType.AddTag);
     }
 
+    // Remove tag
     for (let i = 0; i < results.removeTag.length; i++) {
       const item = results.removeTag[i];
       await db.run(

core/indexing/FullTextSearch.ts

@@ -17,7 +17,8 @@ import {
 
 export class FullTextSearchCodebaseIndex implements CodebaseIndex {
   relativeExpectedTime: number = 0.2;
-  artifactId = "sqliteFts";
+  static artifactId = "sqliteFts";
+  artifactId: string = ChunkCodebaseIndex.artifactId;
 
   private async _createTables(db: DatabaseConnection) {
     await db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(

core/indexing/chunk/ChunkCodebaseIndex.ts

@@ -238,8 +238,4 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
       });
     });
   }
-
-  private formatListPlurality(word: string, length: number): string {
-    return length <= 1 ? word : `${word}s`;
-  }
 }

core/indexing/refreshIndex.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/naming-convention */
 import crypto from "node:crypto";
 import * as fs from "node:fs";
 import { open, type Database } from "sqlite";
@@ -113,11 +114,6 @@ async function getSavedItemsForTag(
   return rows;
 }
 
-interface PathAndOptionalCacheKey {
-  path: string;
-  cacheKey?: string;
-}
-
 enum AddRemoveResultType {
   Add = "add",
   Remove = "remove",
@@ -409,7 +405,7 @@ export async function getComputeDeleteAddRemove(
       for await (const _ of globalCacheIndex.update(
         tag,
         results,
-        async () => { },
+        async () => {},
         repoName,
       )) {
       }

core/test/fixtures.ts

@@ -4,10 +4,16 @@ import FileSystemIde from "../util/filesystem";
 import { TEST_DIR } from "./testUtils/testDir";
 
 export const testIde = new FileSystemIde(TEST_DIR);
+
 export const ideSettingsPromise = testIde.getIdeSettings();
+
+export const testControlPlaneClient = new ControlPlaneClient(
+  Promise.resolve(undefined),
+);
+
 export const testConfigHandler = new ConfigHandler(
   testIde,
   ideSettingsPromise,
   async (text) => {},
-  new ControlPlaneClient(Promise.resolve(undefined)),
+  testControlPlaneClient,
 );

core/test/indexing/ChunkCodebaseIndex.test.ts

@@ -0,0 +1,96 @@
+import { ChunkCodebaseIndex } from "../../indexing/chunk/ChunkCodebaseIndex";
+import { DatabaseConnection, SqliteDb } from "../../indexing/refreshIndex";
+import { IndexResultType } from "../../indexing/types";
+import { testIde } from "../fixtures";
+import { addToTestDir } from "../testUtils/testDir";
+import { jest } from "@jest/globals";
+import {
+  mockFileContents,
+  mockFilename,
+  mockPathAndCacheKey,
+  testContinueServerClient,
+  updateIndexAndAwaitGenerator,
+} from "./utils";
+
+jest.useFakeTimers();
+
+describe("ChunkCodebaseIndex", () => {
+  let index: ChunkCodebaseIndex;
+  let db: DatabaseConnection;
+
+  beforeAll(async () => {
+    const pathSep = await testIde.pathSep();
+
+    index = new ChunkCodebaseIndex(
+      testIde.readFile.bind(testIde),
+      pathSep,
+      testContinueServerClient,
+      1000,
+    );
+
+    addToTestDir([[mockFilename, mockFileContents]]);
+
+    db = await SqliteDb.get();
+  });
+
+  it("should update the index and maintain expected database state, following the same processing order of results as the update method", async () => {
+    const mockMarkComplete = jest
+      .fn()
+      .mockImplementation(() => Promise.resolve()) as any;
+
+    // Compute test
+    await updateIndexAndAwaitGenerator(index, "compute", mockMarkComplete);
+
+    const computeResult = await db.get(
+      "SELECT * FROM chunks WHERE cacheKey = ?",
+      [mockPathAndCacheKey.cacheKey],
+    );
+
+    expect(computeResult).toBeTruthy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.Compute,
+    );
+
+    // AddTag test
+    await updateIndexAndAwaitGenerator(index, "addTag", mockMarkComplete);
+
+    const addTagResult = await db.get(
+      "SELECT * FROM chunk_tags WHERE chunkId = ?",
+      [computeResult.id],
+    );
+
+    expect(addTagResult).toBeTruthy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.AddTag,
+    );
+
+    // RemoveTag test
+    await updateIndexAndAwaitGenerator(index, "removeTag", mockMarkComplete);
+
+    const removeTagResult = await db.get(
+      "SELECT * FROM chunk_tags WHERE id = ?",
+      [addTagResult.id],
+    );
+
+    expect(removeTagResult).toBeFalsy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.RemoveTag,
+    );
+
+    // Delete test
+    await updateIndexAndAwaitGenerator(index, "del", mockMarkComplete);
+
+    const delResult = await db.get("SELECT * FROM chunks WHERE id = ?", [
+      computeResult.id,
+    ]);
+
+    expect(delResult).toBeFalsy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.Delete,
+    );
+  });
+});

core/test/indexing/CodeSnippetsIndex.test.ts

@@ -0,0 +1,115 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { CodeSnippetsCodebaseIndex } from "../../indexing/CodeSnippetsIndex";
+import { DatabaseConnection, SqliteDb } from "../../indexing/refreshIndex";
+import { IndexResultType } from "../../indexing/types";
+import { testIde } from "../fixtures";
+import {
+  insertMockChunks,
+  mockPathAndCacheKey,
+  updateIndexAndAwaitGenerator,
+} from "./utils";
+import { jest } from "@jest/globals";
+
+describe("CodeSnippetsCodebaseIndex", () => {
+  let index: CodeSnippetsCodebaseIndex;
+  let db: DatabaseConnection;
+
+  beforeEach(async () => {
+    db = await SqliteDb.get();
+    index = new CodeSnippetsCodebaseIndex(testIde);
+  });
+
+  it("should update the index and maintain expected database state, following the same processing order of results as the update method", async () => {
+    const mockMarkComplete = jest
+      .fn()
+      .mockImplementation(() => Promise.resolve()) as any;
+
+    const mockSnippet = {
+      title: "",
+      content: "",
+      startLine: 0,
+      endLine: 1,
+    };
+
+    // We mock this fn since currently in testing the directory structure to access the tree-sitter
+    // binaries does not match what is in the release environment.
+    jest
+      .spyOn(CodeSnippetsCodebaseIndex.prototype, "getSnippetsInFile")
+      .mockResolvedValue([mockSnippet]);
+
+    await insertMockChunks();
+
+    // Compute test
+    await updateIndexAndAwaitGenerator(index, "compute", mockMarkComplete);
+
+    const computeResult = await db.get(
+      "SELECT * FROM code_snippets WHERE path = ?",
+      [mockPathAndCacheKey.path],
+    );
+
+    const computeResultTags = await db.get(
+      "SELECT * FROM code_snippets_tags WHERE snippetId = ?",
+      [computeResult.id],
+    );
+
+    expect(computeResult).toBeTruthy();
+    expect(computeResultTags).toBeTruthy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.Compute,
+    );
+
+    // Delete test
+    await updateIndexAndAwaitGenerator(index, "del", mockMarkComplete);
+
+    const delResult = await db.get("SELECT * FROM code_snippets WHERE id = ?", [
+      computeResult.id,
+    ]);
+
+    const delResultTags = await db.get(
+      "SELECT * FROM code_snippets_tags WHERE id = ?",
+      [computeResultTags.id],
+    );
+
+    expect(delResult).toBeFalsy();
+    expect(delResultTags).toBeFalsy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.Delete,
+    );
+
+    // AddTag test
+    await updateIndexAndAwaitGenerator(index, "addTag", mockMarkComplete);
+
+    const addTagResult = await db.get(
+      "SELECT * FROM code_snippets WHERE path = ?",
+      [mockPathAndCacheKey.path],
+    );
+
+    const addTagResultTags = await db.get(
+      "SELECT * FROM code_snippets_tags WHERE snippetId = ?",
+      [addTagResult.id],
+    );
+
+    expect(addTagResult).toBeTruthy();
+    expect(addTagResultTags).toBeTruthy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.AddTag,
+    );
+
+    // RemoveTag test
+    await updateIndexAndAwaitGenerator(index, "removeTag", mockMarkComplete);
+
+    const removeTagResultTag = await db.get(
+      "SELECT * FROM code_snippets_tags WHERE id = ?",
+      [addTagResultTags.id],
+    );
+
+    expect(removeTagResultTag).toBeFalsy();
+    expect(mockMarkComplete).toHaveBeenCalledWith(
+      [mockPathAndCacheKey],
+      IndexResultType.RemoveTag,
+    );
+  });
+});