From ef14caf6ba6a959d4c53144cdf2d4b25759c7e4c Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Sun, 22 Dec 2024 14:33:39 -0800 Subject: [PATCH] provider for sourcegraph references endpoint --- pnpm-lock.yaml | 19 +++ provider/sourcegraph-refs/README.md | 18 +++ provider/sourcegraph-refs/graphql.ts | 154 +++++++++++++++++++ provider/sourcegraph-refs/graphql_blobs.ts | 41 +++++ provider/sourcegraph-refs/graphql_symbols.ts | 99 ++++++++++++ provider/sourcegraph-refs/graphql_usages.ts | 98 ++++++++++++ provider/sourcegraph-refs/index.ts | 117 ++++++++++++++ provider/sourcegraph-refs/package.json | 32 ++++ provider/sourcegraph-refs/tsconfig.json | 11 ++ provider/sourcegraph-refs/vitest.config.ts | 3 + 10 files changed, 592 insertions(+) create mode 100644 provider/sourcegraph-refs/README.md create mode 100644 provider/sourcegraph-refs/graphql.ts create mode 100644 provider/sourcegraph-refs/graphql_blobs.ts create mode 100644 provider/sourcegraph-refs/graphql_symbols.ts create mode 100644 provider/sourcegraph-refs/graphql_usages.ts create mode 100644 provider/sourcegraph-refs/index.ts create mode 100644 provider/sourcegraph-refs/package.json create mode 100644 provider/sourcegraph-refs/tsconfig.json create mode 100644 provider/sourcegraph-refs/vitest.config.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a874611..82abc585 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -711,6 +711,21 @@ importers: specifier: ^1.0.3 version: 1.0.3 + provider/sourcegraph-refs: + dependencies: + '@openctx/provider': + specifier: workspace:* + version: link:../../lib/provider + '@types/lodash': + specifier: ^4.17.13 + version: 4.17.13 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + zod: + specifier: ^3.23.8 + version: 3.23.8 + provider/sourcegraph-search: dependencies: '@openctx/provider': @@ -6816,6 +6831,10 @@ packages: resolution: {integrity: sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg==} dev: true + /@types/lodash@4.17.13: + resolution: {integrity: sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==} + dev: false + /@types/mdast@4.0.3: resolution: {integrity: sha512-LsjtqsyF+d2/yFOYaN22dHZI1Cpwkrj+g06G8+qtUKlhovPW89YhqSnfKtMbkgmEtYpH2gydRNULd6y8mciAFg==} dependencies: diff --git a/provider/sourcegraph-refs/README.md b/provider/sourcegraph-refs/README.md new file mode 100644 index 00000000..5e340d10 --- /dev/null +++ b/provider/sourcegraph-refs/README.md @@ -0,0 +1,18 @@ +# Sourcegraph refs context provider for OpenCtx + +This is a context provider for [OpenCtx](https://openctx.org) that fetches Sourcegraph references for use as context. + +## Usage + +Add the following to your settings in any OpenCtx client: + +```json +"openctx.providers": { + // ...other providers... + "https://openctx.org/npm/@openctx/provider-sourcegraph-refs": { + "sourcegraphEndpoint": "https://sourcegraph.com", + "sourcegraphToken": "$YOUR_TOKEN", + "repositoryNames": ["github.com/sourcegraph/cody"] + } +}, +``` diff --git a/provider/sourcegraph-refs/graphql.ts b/provider/sourcegraph-refs/graphql.ts new file mode 100644 index 00000000..12f29a94 --- /dev/null +++ b/provider/sourcegraph-refs/graphql.ts @@ -0,0 +1,154 @@ +import { escapeRegExp } from 'lodash' +import { BLOB_QUERY, type BlobInfo, BlobResponseSchema } from './graphql_blobs.js' +import { + FUZZY_SYMBOLS_QUERY, + FuzzySymbolsResponseSchema, + type SymbolInfo, + transformToSymbols, +} from './graphql_symbols.js' +import { + USAGES_FOR_SYMBOL_QUERY, + type Usage, + UsagesForSymbolResponseSchema, + transformToUsages, +} from './graphql_usages.js' + +interface APIResponse { + data?: T + errors?: { message: string; path?: string[] }[] +} + +export class SourcegraphGraphQLAPIClient { + constructor( + private readonly endpoint: string, + private readonly token: string, + ) {} + + public async fetchSymbols(query: string, repositories: string[]): Promise { + const response: any | Error = await this.fetchSourcegraphAPI>( + FUZZY_SYMBOLS_QUERY, + { + query: `type:symbol count:30 ${ + repositories.length > 0 ? `repo:^(${repositories.map(escapeRegExp).join('|')})$` : '' + } ${query}`, + }, + ) + + if (isError(response)) { + return response + } + + try { + const validatedData = FuzzySymbolsResponseSchema.parse(response.data) + return transformToSymbols(validatedData) + } catch (error) { + return new Error(`Invalid response format: ${error}`) + } + } + + public async fetchUsages( + repository: string, + path: string, + startLine: number, + startCharacter: number, + endLine: number, + endCharacter: number, + ): Promise { + const response: any | Error = await this.fetchSourcegraphAPI< + APIResponse + >(USAGES_FOR_SYMBOL_QUERY, { + repository, + path, + startLine, + startCharacter, + endLine, + endCharacter, + }) + + if (isError(response)) { + return response + } + + try { + // TODO(beyang): sort or filter by provenance + const validatedData = UsagesForSymbolResponseSchema.parse(response.data) + return transformToUsages(validatedData) + } catch (error) { + return new Error(`Invalid response format: ${error}`) + } + } + + public async fetchBlob({ + repoName, + revspec, + path, + startLine, + endLine, + }: { + repoName: string + revspec: string + path: string + startLine: number + endLine: number + }): Promise { + const response: any | Error = await this.fetchSourcegraphAPI>(BLOB_QUERY, { + repoName, + revspec, + path, + startLine, + endLine, + }) + + if (isError(response)) { + return response + } + + try { + const validatedData = BlobResponseSchema.parse(response.data) + return { + repoName, + revision: revspec, + path: validatedData.repository.commit.blob.path, + range: { + start: { line: startLine, character: 0 }, + end: { line: endLine, character: 0 }, + }, + content: validatedData.repository.commit.blob.content, + } + } catch (error) { + return new Error(`Invalid response format: ${error}`) + } + } + + public async fetchSourcegraphAPI( + query: string, + variables: Record = {}, + ): Promise { + const headers = new Headers() + headers.set('Content-Type', 'application/json; charset=utf-8') + headers.set('User-Agent', 'openctx-sourcegraph-search / 0.0.1') + headers.set('Authorization', `token ${this.token}`) + + const queryName = query.match(/^\s*(?:query|mutation)\s+(\w+)/m)?.[1] ?? 'unknown' + const url = this.endpoint + '/.api/graphql?' + queryName + + return fetch(url, { + method: 'POST', + body: JSON.stringify({ query, variables }), + headers, + }) + .then(verifyResponseCode) + .then(response => response.json() as T) + .catch(error => new Error(`accessing Sourcegraph GraphQL API: ${error} (${url})`)) + } +} + +async function verifyResponseCode(response: Response): Promise { + if (!response.ok) { + const body = await response.text() + throw new Error(`HTTP status code ${response.status}${body ? `: ${body}` : ''}`) + } + return response +} + +export const isError = (value: unknown): value is Error => value instanceof Error diff --git a/provider/sourcegraph-refs/graphql_blobs.ts b/provider/sourcegraph-refs/graphql_blobs.ts new file mode 100644 index 00000000..7078c22a --- /dev/null +++ b/provider/sourcegraph-refs/graphql_blobs.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' + +export interface BlobInfo { + repoName: string + revision: string + path: string + range: { + start: { line: number; character: number } + end: { line: number; character: number } + } + content: string +} + +export const BlobResponseSchema = z.object({ + repository: z.object({ + commit: z.object({ + blob: z.object({ + path: z.string(), + url: z.string(), + languages: z.array(z.string()), + content: z.string(), + }), + }), + }), +}) + +export type BlobResponse = z.infer + +export const BLOB_QUERY = ` +query Blob($repoName: String!, $revspec: String!, $path: String!, $startLine: Int!, $endLine: Int!) { + repository(name: $repoName) { + commit(rev: $revspec) { + blob(path: $path) { + path + url + languages + content(startLine: $startLine, endLine: $endLine) + } + } + } +}` diff --git a/provider/sourcegraph-refs/graphql_symbols.ts b/provider/sourcegraph-refs/graphql_symbols.ts new file mode 100644 index 00000000..159e8926 --- /dev/null +++ b/provider/sourcegraph-refs/graphql_symbols.ts @@ -0,0 +1,99 @@ +import { z } from 'zod' + +export interface SymbolInfo { + name: string + repositoryId: string + repositoryName: string + path: string + range: { + start: { line: number; character: number } + end: { line: number; character: number } + } +} + +export const FuzzySymbolsResponseSchema = z.object({ + search: z.object({ + results: z.object({ + results: z.array( + z.object({ + __typename: z.string(), + file: z.object({ + path: z.string(), + }), + symbols: z.array( + z.object({ + name: z.string(), + location: z.object({ + range: z.object({ + start: z.object({ line: z.number(), character: z.number() }), + end: z.object({ line: z.number(), character: z.number() }), + }), + resource: z.object({ + path: z.string(), + }), + }), + }), + ), + repository: z.object({ + id: z.string(), + name: z.string(), + }), + }), + ), + }), + }), +}) + +export function transformToSymbols(response: z.infer): SymbolInfo[] { + return response.search.results.results.flatMap(result => { + return (result.symbols || []).map(symbol => ({ + name: symbol.name, + repositoryId: result.repository.id, + repositoryName: result.repository.name, + path: symbol.location.resource.path, + range: { + start: { + line: symbol.location.range.start.line, + character: symbol.location.range.start.character, + }, + end: { + line: symbol.location.range.end.line, + character: symbol.location.range.end.character, + }, + }, + })) + }) +} + +export const FUZZY_SYMBOLS_QUERY = ` +query FuzzySymbols($query: String!) { + search(patternType: regexp, query: $query) { + results { + results { + ... on FileMatch { + __typename + file { + path + } + symbols { + name + location { + range { + start { line, character} + end { line, character } + } + resource { + path + } + } + } + repository { + id + name + } + } + } + } + } +} +` diff --git a/provider/sourcegraph-refs/graphql_usages.ts b/provider/sourcegraph-refs/graphql_usages.ts new file mode 100644 index 00000000..a03b7d2c --- /dev/null +++ b/provider/sourcegraph-refs/graphql_usages.ts @@ -0,0 +1,98 @@ +import { z } from 'zod' + +export interface Usage { + repoName: string + revision: string + path: string + range: { + start: { line: number; character: number } + end: { line: number; character: number } + } + surroundingContent: string +} + +export const UsagesForSymbolResponseSchema = z.object({ + usagesForSymbol: z.object({ + nodes: z.array( + z.object({ + symbol: z + .object({ + name: z.string(), + }) + .nullable(), + usageKind: z.string(), + provenance: z.string(), + surroundingContent: z.string(), + usageRange: z.object({ + repository: z.string(), + revision: z.string(), + path: z.string(), + range: z.object({ + start: z.object({ + line: z.number(), + character: z.number(), + }), + end: z.object({ + line: z.number(), + character: z.number(), + }), + }), + }), + }), + ), + }), +}) +export function transformToUsages(response: z.infer): Usage[] { + return response.usagesForSymbol.nodes.map(node => ({ + repoName: node.usageRange.repository, + revision: node.usageRange.revision, + path: node.usageRange.path, + range: { + start: { + line: node.usageRange.range.start.line, + character: node.usageRange.range.start.character, + }, + end: { + line: node.usageRange.range.end.line, + character: node.usageRange.range.end.character, + }, + }, + surroundingContent: node.surroundingContent, + })) +} + +export const USAGES_FOR_SYMBOL_QUERY = ` +query UsagesForSymbol($repository: String!, $path: String!, $startLine: Int!, $startCharacter: Int!, $endLine: Int!, $endCharacter: Int!) { + usagesForSymbol( + range: { + repository: $repository, + path: $path, + start: { + line: $startLine, + character: $startCharacter + }, + end: { + line: $endLine, + character: $endCharacter + } + } + ) { + nodes { + symbol { + name + } + usageKind + provenance + surroundingContent + usageRange { + repository + revision + path + range { + start { line character } + end { line character } + } + } + } + } +}` diff --git a/provider/sourcegraph-refs/index.ts b/provider/sourcegraph-refs/index.ts new file mode 100644 index 00000000..9097bf3c --- /dev/null +++ b/provider/sourcegraph-refs/index.ts @@ -0,0 +1,117 @@ +import type { + ItemsParams, + ItemsResult, + MentionsParams, + MentionsResult, + MetaParams, + MetaResult, + Provider, +} from '@openctx/provider' +import { z } from 'zod' +import { SourcegraphGraphQLAPIClient, isError } from './graphql.js' +import type { SymbolInfo } from './graphql_symbols.js' + +const settingsSchema = z.object({ + sourcegraphEndpoint: z.string(), + sourcegraphToken: z.string(), + repositoryNames: z.string().array(), +}) + +/** + * An OpenCtx provider that fetches the content of a URL and provides it as an item. + */ +class SourcegraphRefsProvider implements Provider> { + private graphqlClient: SourcegraphGraphQLAPIClient | undefined + private repositories: string[] = [] + private sourcegraphEndpoint = '' + + public meta(_params: MetaParams, settings: z.infer): MetaResult { + this.graphqlClient = new SourcegraphGraphQLAPIClient( + settings.sourcegraphEndpoint, + settings.sourcegraphToken, + ) + this.sourcegraphEndpoint = settings.sourcegraphEndpoint + this.repositories = settings.repositoryNames + return { + name: 'Sourcegraph usages', + mentions: { label: 'Type a symbol name' }, + annotations: { selectors: [] }, + } + } + + async mentions(params: MentionsParams): Promise { + if (!this.graphqlClient) { + return [] + } + const symbols = await this.graphqlClient.fetchSymbols(params.query ?? '', this.repositories) + if (isError(symbols)) { + return [] + } + const mentions = symbols.map(s => ({ + title: s.name, + uri: `${s.path}@L${s.range.start.line + 1}`, + data: { + symbol: s, + }, + })) + return mentions + } + + async items(params: ItemsParams): Promise { + if (!this.graphqlClient) { + return [] + } + const mention = params.mention + if (!mention?.data?.symbol) { + return [] + } + + const symbol: SymbolInfo | undefined = mention.data.symbol as SymbolInfo + if (!symbol) { + return [] + } + + const usages = await this.graphqlClient.fetchUsages( + symbol.repositoryName, + symbol.path, + symbol.range.start.line, + symbol.range.start.character, + symbol.range.end.line, + symbol.range.end.character, + ) + + if (isError(usages)) { + console.error('Error fetching usages:', usages) + return [] + } + + const graphqlClient = this.graphqlClient + return ( + await Promise.all( + usages.flatMap(async usage => { + const blob = await graphqlClient.fetchBlob({ + repoName: usage.repoName, + revspec: usage.revision, + path: usage.path, + startLine: Math.max(1, usage.range.start.line + 1 - 5), + endLine: usage.range.end.line + 1 + 5, + }) + if (isError(blob)) { + console.error('Error fetching blob:', blob) + return [] + } + return { + title: usage.path, + url: `${this.sourcegraphEndpoint}/${usage.repoName}@${usage.revision}/-/blob/${usage.path}?L${usage.range.start.line + 1}-${usage.range.end.line + 1}`, + ai: { + content: blob.content, + }, + } + }), + ) + ).flat() + } +} + +const sourcegraphRefsProvider = new SourcegraphRefsProvider() +export default sourcegraphRefsProvider diff --git a/provider/sourcegraph-refs/package.json b/provider/sourcegraph-refs/package.json new file mode 100644 index 00000000..f7fb3277 --- /dev/null +++ b/provider/sourcegraph-refs/package.json @@ -0,0 +1,32 @@ +{ + "name": "@openctx/provider-sourcegraph-refs", + "version": "0.0.13", + "description": "Use Sourcegraph references as context (OpenCtx provider)", + "license": "Apache-2.0", + "homepage": "https://openctx.org/docs/providers/sourcegraph-refs", + "repository": { + "type": "git", + "url": "https://github.com/sourcegraph/openctx", + "directory": "provider/sourcegraph-refs" + }, + "type": "module", + "main": "dist/bundle.js", + "types": "dist/index.d.ts", + "files": [ + "dist/bundle.js", + "dist/index.d.ts" + ], + "sideEffects": false, + "scripts": { + "bundle": "tsc --build && esbuild --log-level=error --bundle --format=esm --outfile=dist/bundle.js index.ts", + "prepublishOnly": "tsc --build --clean && npm run --silent bundle", + "test": "vitest", + "watch": "tsc --build --watch & esbuild --log-level=error --bundle --format=esm --outfile=dist/bundle.js --watch index.ts" + }, + "dependencies": { + "@openctx/provider": "workspace:*", + "@types/lodash": "^4.17.13", + "lodash": "^4.17.21", + "zod": "^3.23.8" + } +} \ No newline at end of file diff --git a/provider/sourcegraph-refs/tsconfig.json b/provider/sourcegraph-refs/tsconfig.json new file mode 100644 index 00000000..a1d94187 --- /dev/null +++ b/provider/sourcegraph-refs/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../.config/tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "dist", + "lib": ["ESNext"] + }, + "include": ["*.ts"], + "exclude": ["dist", "vitest.config.ts"], + "references": [{ "path": "../../lib/provider" }] +} diff --git a/provider/sourcegraph-refs/vitest.config.ts b/provider/sourcegraph-refs/vitest.config.ts new file mode 100644 index 00000000..abed6b21 --- /dev/null +++ b/provider/sourcegraph-refs/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({})