diff --git a/core/indexing/CodeSnippetsIndex.ts b/core/indexing/CodeSnippetsIndex.ts index e431dfdf83..f16c5137a6 100644 --- a/core/indexing/CodeSnippetsIndex.ts +++ b/core/indexing/CodeSnippetsIndex.ts @@ -100,26 +100,30 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { contents: string, ): Promise<(ChunkWithoutID & { title: string })[]> { const parser = await getParserForFile(filepath); + if (!parser) { return []; } + const ast = parser.parse(contents); const query = await getQueryForFile(filepath, TSQueryType.CodeSnippets); const matches = query?.matches(ast.rootNode); - return ( - matches?.flatMap((match) => { - const node = match.captures[0].node; - const title = match.captures[1].node.text; - const results = { - title, - content: node.text, - startLine: node.startPosition.row, - endLine: node.endPosition.row, - }; - return results; - }) ?? [] - ); + if (!matches) { + return []; + } + + return matches.flatMap((match) => { + const node = match.captures[0].node; + const title = match.captures[1].node.text; + const results = { + title, + content: node.text, + startLine: node.startPosition.row, + endLine: node.endPosition.row, + }; + return results; + }); } async *update( @@ -132,6 +136,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { await CodeSnippetsCodebaseIndex._createTables(db); const tagString = tagToString(tag); + // Compute for (let i = 0; i < results.compute.length; i++) { const compute = results.compute[i]; @@ -174,6 +179,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { markComplete([compute], IndexResultType.Compute); } + // Delete for (let i = 0; i < results.del.length; i++) { const del = results.del[i]; const deleted = await db.run( @@ -186,6 +192,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { markComplete([del], IndexResultType.Delete); } + // Add tag for (let i = 0; i < results.addTag.length; i++) { const addTag = results.addTag[i]; let snippets: (ChunkWithoutID & { title: string })[] = []; @@ -220,6 +227,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { markComplete([results.addTag[i]], IndexResultType.AddTag); } + // Remove tag for (let i = 0; i < results.removeTag.length; i++) { const item = results.removeTag[i]; await db.run( diff --git a/core/indexing/FullTextSearch.ts b/core/indexing/FullTextSearch.ts index c640eae203..982886d457 100644 --- a/core/indexing/FullTextSearch.ts +++ b/core/indexing/FullTextSearch.ts @@ -17,7 +17,8 @@ import { export class FullTextSearchCodebaseIndex implements CodebaseIndex { relativeExpectedTime: number = 0.2; - artifactId = "sqliteFts"; + static artifactId = "sqliteFts"; + artifactId: string = ChunkCodebaseIndex.artifactId; private async _createTables(db: DatabaseConnection) { await db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5( diff --git a/core/indexing/chunk/ChunkCodebaseIndex.ts b/core/indexing/chunk/ChunkCodebaseIndex.ts index 498e5144e0..671ce31e6e 100644 --- a/core/indexing/chunk/ChunkCodebaseIndex.ts +++ b/core/indexing/chunk/ChunkCodebaseIndex.ts @@ -238,8 +238,4 @@ export class ChunkCodebaseIndex implements CodebaseIndex { }); }); } - - private formatListPlurality(word: string, length: number): string { - return length <= 1 ? word : `${word}s`; - } } diff --git a/core/indexing/refreshIndex.ts b/core/indexing/refreshIndex.ts index 10fe556054..7cfeae0f98 100644 --- a/core/indexing/refreshIndex.ts +++ b/core/indexing/refreshIndex.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import crypto from "node:crypto"; import * as fs from "node:fs"; import { open, type Database } from "sqlite"; @@ -113,11 +114,6 @@ async function getSavedItemsForTag( return rows; } -interface PathAndOptionalCacheKey { - path: string; - cacheKey?: string; -} - enum AddRemoveResultType { Add = "add", Remove = "remove", @@ -409,7 +405,7 @@ export async function getComputeDeleteAddRemove( for await (const _ of globalCacheIndex.update( tag, results, - async () => { }, + async () => {}, repoName, )) { } diff --git a/core/test/fixtures.ts b/core/test/fixtures.ts index 97c1815b06..51f974feaa 100644 --- a/core/test/fixtures.ts +++ b/core/test/fixtures.ts @@ -4,10 +4,16 @@ import FileSystemIde from "../util/filesystem"; import { TEST_DIR } from "./testUtils/testDir"; export const testIde = new FileSystemIde(TEST_DIR); + export const ideSettingsPromise = testIde.getIdeSettings(); + +export const testControlPlaneClient = new ControlPlaneClient( + Promise.resolve(undefined), +); + export const testConfigHandler = new ConfigHandler( testIde, ideSettingsPromise, async (text) => {}, - new ControlPlaneClient(Promise.resolve(undefined)), + testControlPlaneClient, ); diff --git a/core/test/indexing/ChunkCodebaseIndex.test.ts b/core/test/indexing/ChunkCodebaseIndex.test.ts new file mode 100644 index 0000000000..3c8693beb0 --- /dev/null +++ b/core/test/indexing/ChunkCodebaseIndex.test.ts @@ -0,0 +1,96 @@ +import { ChunkCodebaseIndex } from "../../indexing/chunk/ChunkCodebaseIndex"; +import { DatabaseConnection, SqliteDb } from "../../indexing/refreshIndex"; +import { IndexResultType } from "../../indexing/types"; +import { testIde } from "../fixtures"; +import { addToTestDir } from "../testUtils/testDir"; +import { jest } from "@jest/globals"; +import { + mockFileContents, + mockFilename, + mockPathAndCacheKey, + testContinueServerClient, + updateIndexAndAwaitGenerator, +} from "./utils"; + +jest.useFakeTimers(); + +describe("ChunkCodebaseIndex", () => { + let index: ChunkCodebaseIndex; + let db: DatabaseConnection; + + beforeAll(async () => { + const pathSep = await testIde.pathSep(); + + index = new ChunkCodebaseIndex( + testIde.readFile.bind(testIde), + pathSep, + testContinueServerClient, + 1000, + ); + + addToTestDir([[mockFilename, mockFileContents]]); + + db = await SqliteDb.get(); + }); + + it("should update the index and maintain expected database state, following the same processing order of results as the update method", async () => { + const mockMarkComplete = jest + .fn() + .mockImplementation(() => Promise.resolve()) as any; + + // Compute test + await updateIndexAndAwaitGenerator(index, "compute", mockMarkComplete); + + const computeResult = await db.get( + "SELECT * FROM chunks WHERE cacheKey = ?", + [mockPathAndCacheKey.cacheKey], + ); + + expect(computeResult).toBeTruthy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Compute, + ); + + // AddTag test + await updateIndexAndAwaitGenerator(index, "addTag", mockMarkComplete); + + const addTagResult = await db.get( + "SELECT * FROM chunk_tags WHERE chunkId = ?", + [computeResult.id], + ); + + expect(addTagResult).toBeTruthy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.AddTag, + ); + + // RemoveTag test + await updateIndexAndAwaitGenerator(index, "removeTag", mockMarkComplete); + + const removeTagResult = await db.get( + "SELECT * FROM chunk_tags WHERE id = ?", + [addTagResult.id], + ); + + expect(removeTagResult).toBeFalsy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.RemoveTag, + ); + + // Delete test + await updateIndexAndAwaitGenerator(index, "del", mockMarkComplete); + + const delResult = await db.get("SELECT * FROM chunks WHERE id = ?", [ + computeResult.id, + ]); + + expect(delResult).toBeFalsy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Delete, + ); + }); +}); diff --git a/core/test/indexing/CodeSnippetsIndex.test.ts b/core/test/indexing/CodeSnippetsIndex.test.ts new file mode 100644 index 0000000000..2eabaa93fc --- /dev/null +++ b/core/test/indexing/CodeSnippetsIndex.test.ts @@ -0,0 +1,115 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { CodeSnippetsCodebaseIndex } from "../../indexing/CodeSnippetsIndex"; +import { DatabaseConnection, SqliteDb } from "../../indexing/refreshIndex"; +import { IndexResultType } from "../../indexing/types"; +import { testIde } from "../fixtures"; +import { + insertMockChunks, + mockPathAndCacheKey, + updateIndexAndAwaitGenerator, +} from "./utils"; +import { jest } from "@jest/globals"; + +describe("CodeSnippetsCodebaseIndex", () => { + let index: CodeSnippetsCodebaseIndex; + let db: DatabaseConnection; + + beforeEach(async () => { + db = await SqliteDb.get(); + index = new CodeSnippetsCodebaseIndex(testIde); + }); + + it("should update the index and maintain expected database state, following the same processing order of results as the update method", async () => { + const mockMarkComplete = jest + .fn() + .mockImplementation(() => Promise.resolve()) as any; + + const mockSnippet = { + title: "", + content: "", + startLine: 0, + endLine: 1, + }; + + // We mock this fn since currently in testing the directory structure to access the tree-sitter + // binaries does not match what is in the release environment. + jest + .spyOn(CodeSnippetsCodebaseIndex.prototype, "getSnippetsInFile") + .mockResolvedValue([mockSnippet]); + + await insertMockChunks(); + + // Compute test + await updateIndexAndAwaitGenerator(index, "compute", mockMarkComplete); + + const computeResult = await db.get( + "SELECT * FROM code_snippets WHERE path = ?", + [mockPathAndCacheKey.path], + ); + + const computeResultTags = await db.get( + "SELECT * FROM code_snippets_tags WHERE snippetId = ?", + [computeResult.id], + ); + + expect(computeResult).toBeTruthy(); + expect(computeResultTags).toBeTruthy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Compute, + ); + + // Delete test + await updateIndexAndAwaitGenerator(index, "del", mockMarkComplete); + + const delResult = await db.get("SELECT * FROM code_snippets WHERE id = ?", [ + computeResult.id, + ]); + + const delResultTags = await db.get( + "SELECT * FROM code_snippets_tags WHERE id = ?", + [computeResultTags.id], + ); + + expect(delResult).toBeFalsy(); + expect(delResultTags).toBeFalsy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Delete, + ); + + // AddTag test + await updateIndexAndAwaitGenerator(index, "addTag", mockMarkComplete); + + const addTagResult = await db.get( + "SELECT * FROM code_snippets WHERE path = ?", + [mockPathAndCacheKey.path], + ); + + const addTagResultTags = await db.get( + "SELECT * FROM code_snippets_tags WHERE snippetId = ?", + [addTagResult.id], + ); + + expect(addTagResult).toBeTruthy(); + expect(addTagResultTags).toBeTruthy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.AddTag, + ); + + // RemoveTag test + await updateIndexAndAwaitGenerator(index, "removeTag", mockMarkComplete); + + const removeTagResultTag = await db.get( + "SELECT * FROM code_snippets_tags WHERE id = ?", + [addTagResultTags.id], + ); + + expect(removeTagResultTag).toBeFalsy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.RemoveTag, + ); + }); +}); diff --git a/core/test/indexing/FullTextSearchCodebaseIndex.test.ts b/core/test/indexing/FullTextSearchCodebaseIndex.test.ts new file mode 100644 index 0000000000..cb27ee9202 --- /dev/null +++ b/core/test/indexing/FullTextSearchCodebaseIndex.test.ts @@ -0,0 +1,82 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { FullTextSearchCodebaseIndex } from "../../indexing/FullTextSearch"; +import { DatabaseConnection, SqliteDb } from "../../indexing/refreshIndex"; +import { IndexResultType } from "../../indexing/types"; +import { + insertMockChunks, + mockPathAndCacheKey, + updateIndexAndAwaitGenerator, +} from "./utils"; +import { jest } from "@jest/globals"; + +describe("FullTextSearchCodebaseIndex", () => { + let index: FullTextSearchCodebaseIndex; + let db: DatabaseConnection; + + beforeEach(async () => { + db = await SqliteDb.get(); + index = new FullTextSearchCodebaseIndex(); + }); + + it("should update the index and maintain expected database state, following the same processing order of results as the update method", async () => { + const mockMarkComplete = jest + .fn() + .mockImplementation(() => Promise.resolve()) as any; + + await insertMockChunks(); + + // Compute test + await updateIndexAndAwaitGenerator(index, "compute", mockMarkComplete); + + const computeResult = await db.get("SELECT * FROM fts WHERE path = ?", [ + mockPathAndCacheKey.path, + ]); + + const computeResultMetadata = await db.get( + "SELECT * FROM fts_metadata WHERE path = ?", + [mockPathAndCacheKey.path], + ); + + expect(computeResult).toBeTruthy(); + expect(computeResultMetadata).toBeTruthy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Compute, + ); + + // AddTag test - currently, we don't do anything other than mark complete + await updateIndexAndAwaitGenerator(index, "addTag", mockMarkComplete); + + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.AddTag, + ); + + // RemoveTag test - currently, we don't do anything other than mark complete + await updateIndexAndAwaitGenerator(index, "removeTag", mockMarkComplete); + + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.RemoveTag, + ); + + // Delete test + await updateIndexAndAwaitGenerator(index, "del", mockMarkComplete); + + const delResult = await db.get("SELECT * FROM fts WHERE path = ?", [ + mockPathAndCacheKey.cacheKey, + ]); + + const delResultMetadata = await db.get( + "SELECT * FROM fts_metadata WHERE path = ?", + [mockPathAndCacheKey.path], + ); + + expect(delResult).toBeFalsy(); + expect(delResultMetadata).toBeFalsy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Delete, + ); + }); +}); diff --git a/core/test/indexing/LanceDbIndex.test.ts b/core/test/indexing/LanceDbIndex.test.ts new file mode 100644 index 0000000000..1cf1ec8002 --- /dev/null +++ b/core/test/indexing/LanceDbIndex.test.ts @@ -0,0 +1,94 @@ +import { ChunkCodebaseIndex } from "../../indexing/chunk/ChunkCodebaseIndex"; +import { DatabaseConnection, SqliteDb } from "../../indexing/refreshIndex"; +import { IndexResultType } from "../../indexing/types"; +import { testIde } from "../fixtures"; +import { addToTestDir } from "../testUtils/testDir"; +import { jest } from "@jest/globals"; +import { + testContinueServerClient, + updateIndexAndAwaitGenerator, +} from "./utils"; +import { LanceDbIndex } from "../../indexing/LanceDbIndex"; + +jest.useFakeTimers(); + +describe("ChunkCodebaseIndex", () => { + let index: ChunkCodebaseIndex; + let db: DatabaseConnection; + + beforeAll(async () => { + const pathSep = await testIde.pathSep(); + + index = new LanceDbIndex( + undefined, + testIde.readFile.bind(testIde), + pathSep, + testContinueServerClient, + ); + + addToTestDir([[mockFilename, mockFileContents]]); + + db = await SqliteDb.get(); + }); + + it("should update the index and maintain expected database state, following the same processing order of results as the update method", async () => { + const mockMarkComplete = jest + .fn() + .mockImplementation(() => Promise.resolve()) as any; + + // Compute test + await updateIndexAndAwaitGenerator(index, "compute", mockMarkComplete); + + const computeResult = await db.get( + "SELECT * FROM chunks WHERE cacheKey = ?", + [mockPathAndCacheKey.cacheKey], + ); + + expect(computeResult).toBeTruthy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Compute, + ); + + // AddTag test + await updateIndexAndAwaitGenerator(index, "addTag", mockMarkComplete); + + const addTagResult = await db.get( + "SELECT * FROM chunk_tags WHERE chunkId = ?", + [computeResult.id], + ); + + expect(addTagResult).toBeTruthy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.AddTag, + ); + + // RemoveTag test + await updateIndexAndAwaitGenerator(index, "removeTag", mockMarkComplete); + + const removeTagResult = await db.get( + "SELECT * FROM chunk_tags WHERE id = ?", + [addTagResult.id], + ); + + expect(removeTagResult).toBeFalsy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.RemoveTag, + ); + + // Delete test + await updateIndexAndAwaitGenerator(index, "del", mockMarkComplete); + + const delResult = await db.get("SELECT * FROM chunks WHERE id = ?", [ + computeResult.id, + ]); + + expect(delResult).toBeFalsy(); + expect(mockMarkComplete).toHaveBeenCalledWith( + [mockPathAndCacheKey], + IndexResultType.Delete, + ); + }); +}); diff --git a/core/test/indexing/utils.ts b/core/test/indexing/utils.ts new file mode 100644 index 0000000000..e79a79b272 --- /dev/null +++ b/core/test/indexing/utils.ts @@ -0,0 +1,82 @@ +import { ChunkCodebaseIndex } from "../../indexing/chunk/ChunkCodebaseIndex"; +import { IContinueServerClient } from "../../continueServer/interface"; +import { testIde } from "../fixtures"; +import { addToTestDir, TEST_DIR } from "../testUtils/testDir"; +import { IndexTag } from "../.."; +import { CodebaseIndex, RefreshIndexResults } from "../../indexing/types"; +import { jest } from "@jest/globals"; +import { tagToString } from "../../indexing/refreshIndex"; + +export const mockFilename = "test.py"; +export const mockPathAndCacheKey = { + path: `${TEST_DIR}/${mockFilename}`, + cacheKey: "abc123", +}; +export const mockFileContents = `\ +def main(): + print("Hello, world!") + +class Foo: + def __init__(self, bar: str): + self.bar = bar +`; + +export const mockTag: IndexTag = { + branch: "main", + directory: "/", + artifactId: "artifactId", +}; + +export const mockTagString = tagToString(mockTag); + +export const testContinueServerClient = { + connected: false, + getFromIndexCache: jest.fn(), +} as unknown as IContinueServerClient; + +const mockContinueServerClient = { + connected: false, + getFromIndexCache: jest.fn(), +} as unknown as IContinueServerClient; + +const mockResults: RefreshIndexResults = { + compute: [], + addTag: [], + removeTag: [], + del: [], +}; + +const mockMarkComplete = jest + .fn() + .mockImplementation(() => Promise.resolve()) as any; + +export async function insertMockChunks() { + const pathSep = await testIde.pathSep(); + + const index = new ChunkCodebaseIndex( + testIde.readFile.bind(testIde), + pathSep, + mockContinueServerClient, + 1000, + ); + + addToTestDir([[mockFilename, mockFileContents]]); + + await updateIndexAndAwaitGenerator(index, "compute", mockMarkComplete); + await updateIndexAndAwaitGenerator(index, "addTag", mockMarkComplete); +} + +export async function updateIndexAndAwaitGenerator( + index: CodebaseIndex, + resultType: keyof RefreshIndexResults, + mockMarkComplete: any, +) { + const computeGenerator = index.update( + mockTag, + { ...mockResults, [resultType]: [mockPathAndCacheKey] }, + mockMarkComplete, + "test-repo", + ); + + while (!(await computeGenerator.next()).done) {} +} diff --git a/core/test/testUtils/testDir.ts b/core/test/testUtils/testDir.ts index 557906cef4..d6bc462dc8 100644 --- a/core/test/testUtils/testDir.ts +++ b/core/test/testUtils/testDir.ts @@ -1,8 +1,9 @@ import fs from "fs"; import path from "path"; +import os from "os"; // Want this outside of the git repository so we can change branches in tests -export const TEST_DIR = path.join(__dirname, "..", "..", "testDir"); +export const TEST_DIR = path.join(os.tmpdir(), "testDir"); export function setUpTestDir() { if (fs.existsSync(TEST_DIR)) {