From de331d602325aae315cf80be1b3b0fd96044fe28 Mon Sep 17 00:00:00 2001 From: Valery Bugakov Date: Tue, 4 Mar 2025 11:30:33 +0800 Subject: [PATCH 1/4] chore(auto-edit): capture request/response metadata from auto-edit API adapters --- lib/shared/src/inferenceClient/misc.ts | 12 ++++ vscode/src/autoedits/adapters/base.ts | 25 +++++++- .../autoedits/adapters/cody-gateway.test.ts | 3 + vscode/src/autoedits/adapters/cody-gateway.ts | 59 ++++++++++++------ .../src/autoedits/adapters/fireworks.test.ts | 9 ++- vscode/src/autoedits/adapters/fireworks.ts | 24 ++++++-- vscode/src/autoedits/adapters/openai.ts | 14 +++-- .../adapters/sourcegraph-chat.test.ts | 2 +- .../autoedits/adapters/sourcegraph-chat.ts | 12 +++- .../adapters/sourcegraph-completions.test.ts | 2 +- .../adapters/sourcegraph-completions.ts | 61 +++++++++++++++---- vscode/src/autoedits/adapters/utils.ts | 44 ++++++++++--- .../src/autoedits/autoedits-provider.test.ts | 16 ++++- vscode/src/autoedits/autoedits-provider.ts | 22 +++++-- vscode/src/autoedits/test-helpers.ts | 22 +++++-- vscode/src/completions/default-client.ts | 13 +++- 16 files changed, 272 insertions(+), 68 deletions(-) diff --git a/lib/shared/src/inferenceClient/misc.ts b/lib/shared/src/inferenceClient/misc.ts index 7b436b5f5a31..c73b0d681c88 100644 --- a/lib/shared/src/inferenceClient/misc.ts +++ b/lib/shared/src/inferenceClient/misc.ts @@ -42,6 +42,18 @@ export type CompletionResponseWithMetaData = { * extract metadata required for analytics in one place. */ response?: BrowserOrNodeResponse + /** + * Optional request headers sent to the model API + */ + requestHeaders?: Record + /** + * URL used to make the request to the model API + */ + requestUrl?: string + /** + * Optional request body sent to the model API + */ + requestBody?: any } } diff --git a/vscode/src/autoedits/adapters/base.ts b/vscode/src/autoedits/adapters/base.ts index efc9332df907..828cae00c807 100644 --- a/vscode/src/autoedits/adapters/base.ts +++ b/vscode/src/autoedits/adapters/base.ts @@ -1,7 +1,28 @@ -import type { PromptString } from '@sourcegraph/cody-shared' +import type { CodeCompletionsParams, PromptString } from '@sourcegraph/cody-shared' +import type { AutoeditsRequestBody } from './utils' + +export interface ModelResponse { + prediction: string + /** URL used to make the request to the model API */ + requestUrl: string + /** Response headers received from the model API */ + responseHeaders: Record + /** Optional request headers sent to the model API */ + requestHeaders?: Record + /** + * Optional request body sent to the model API + * TODO: update to proper types from different adapters. + */ + requestBody?: AutoeditsRequestBody | CodeCompletionsParams + /** + * Optional full response body received from the model API + * This is propagated to the analytics logger for debugging purposes + */ + responseBody?: any +} export interface AutoeditsModelAdapter { - getModelResponse(args: AutoeditModelOptions): Promise + getModelResponse(args: AutoeditModelOptions): Promise } /** diff --git a/vscode/src/autoedits/adapters/cody-gateway.test.ts b/vscode/src/autoedits/adapters/cody-gateway.test.ts index 2d43d2a10905..cba8d017f031 100644 --- a/vscode/src/autoedits/adapters/cody-gateway.test.ts +++ b/vscode/src/autoedits/adapters/cody-gateway.test.ts @@ -50,6 +50,7 @@ describe('CodyGatewayAdapter', () => { // Mock successful response mockFetch.mockResolvedValueOnce({ status: 200, + headers: new Headers(), json: () => Promise.resolve({ choices: [{ message: { content: 'response' } }] }), }) @@ -90,6 +91,7 @@ describe('CodyGatewayAdapter', () => { mockFetch.mockResolvedValueOnce({ status: 200, + headers: new Headers(), json: () => Promise.resolve({ choices: [{ text: 'response' }] }), }) @@ -116,6 +118,7 @@ describe('CodyGatewayAdapter', () => { it('handles error responses correctly', async () => { mockFetch.mockResolvedValueOnce({ status: 400, + headers: new Headers(), text: () => Promise.resolve('Bad Request'), }) diff --git a/vscode/src/autoedits/adapters/cody-gateway.ts b/vscode/src/autoedits/adapters/cody-gateway.ts index 8a1cda6ddc63..73d799f16ce6 100644 --- a/vscode/src/autoedits/adapters/cody-gateway.ts +++ b/vscode/src/autoedits/adapters/cody-gateway.ts @@ -2,27 +2,47 @@ import { currentResolvedConfig, dotcomTokenToGatewayToken } from '@sourcegraph/c import { autoeditsOutputChannelLogger } from '../output-channel-logger' -import type { AutoeditModelOptions, AutoeditsModelAdapter } from './base' +import type { AutoeditModelOptions, AutoeditsModelAdapter, ModelResponse } from './base' import { + type AutoeditsRequestBody, + type FireworksChatModelRequestParams, type FireworksCompatibleRequestParams, + type FireworksCompletionModelRequestParams, getMaxOutputTokensForAutoedits, getModelResponse, getOpenaiCompatibleChatPrompt, } from './utils' export class CodyGatewayAdapter implements AutoeditsModelAdapter { - public async getModelResponse(options: AutoeditModelOptions): Promise { + public async getModelResponse(options: AutoeditModelOptions): Promise { const headers = { 'X-Sourcegraph-Feature': 'code_completions', } const body = this.getMessageBody(options) try { const apiKey = await this.getApiKey() - const response = await getModelResponse(options.url, body, apiKey, headers) + const { data, requestHeaders, responseHeaders, url } = await getModelResponse( + options.url, + JSON.stringify(body), + apiKey, + headers + ) + + let prediction: string if (options.isChatModel) { - return response.choices[0].message.content + prediction = data.choices[0].message.content + } else { + prediction = data.choices[0].text + } + + return { + prediction, + responseHeaders, + requestHeaders, + requestBody: body, + requestUrl: url, + responseBody: data, } - return response.choices[0].text } catch (error) { autoeditsOutputChannelLogger.logError('getModelResponse', 'Error calling Cody Gateway:', { verbose: error, @@ -46,9 +66,9 @@ export class CodyGatewayAdapter implements AutoeditsModelAdapter { return fastPathAccessToken } - private getMessageBody(options: AutoeditModelOptions): string { + private getMessageBody(options: AutoeditModelOptions): AutoeditsRequestBody { const maxTokens = getMaxOutputTokensForAutoedits(options.codeToRewrite) - const body: FireworksCompatibleRequestParams = { + const baseBody: FireworksCompatibleRequestParams = { stream: false, model: options.model, temperature: 0.1, @@ -63,15 +83,20 @@ export class CodyGatewayAdapter implements AutoeditsModelAdapter { rewrite_speculation: true, user: options.userId || undefined, } - const request = options.isChatModel - ? { - ...body, - messages: getOpenaiCompatibleChatPrompt({ - systemMessage: options.prompt.systemMessage, - userMessage: options.prompt.userMessage, - }), - } - : { ...body, prompt: options.prompt.userMessage } - return JSON.stringify(request) + + if (options.isChatModel) { + return { + ...baseBody, + messages: getOpenaiCompatibleChatPrompt({ + systemMessage: options.prompt.systemMessage, + userMessage: options.prompt.userMessage, + }), + } as FireworksChatModelRequestParams + } + + return { + ...baseBody, + prompt: options.prompt.userMessage, + } as FireworksCompletionModelRequestParams } } diff --git a/vscode/src/autoedits/adapters/fireworks.test.ts b/vscode/src/autoedits/adapters/fireworks.test.ts index 0e51c0e5cc58..1a3e9754a117 100644 --- a/vscode/src/autoedits/adapters/fireworks.test.ts +++ b/vscode/src/autoedits/adapters/fireworks.test.ts @@ -45,6 +45,7 @@ describe('FireworksAdapter', () => { it('sends correct request parameters for chat model', async () => { mockFetch.mockResolvedValueOnce({ status: 200, + headers: new Headers(), json: () => Promise.resolve({ choices: [{ message: { content: 'response' } }] }), }) @@ -82,6 +83,7 @@ describe('FireworksAdapter', () => { mockFetch.mockResolvedValueOnce({ status: 200, + headers: new Headers(), json: () => Promise.resolve({ choices: [{ text: 'response' }] }), }) @@ -108,6 +110,7 @@ describe('FireworksAdapter', () => { it('handles error responses correctly', async () => { mockFetch.mockResolvedValueOnce({ status: 400, + headers: new Headers(), text: () => Promise.resolve('Bad Request'), }) @@ -118,11 +121,12 @@ describe('FireworksAdapter', () => { const expectedResponse = 'modified code' mockFetch.mockResolvedValueOnce({ status: 200, + headers: new Headers(), json: () => Promise.resolve({ choices: [{ message: { content: expectedResponse } }] }), }) const response = await adapter.getModelResponse(options) - expect(response).toBe(expectedResponse) + expect(response.prediction).toBe(expectedResponse) }) it('returns correct response for completions model', async () => { @@ -131,10 +135,11 @@ describe('FireworksAdapter', () => { mockFetch.mockResolvedValueOnce({ status: 200, + headers: new Headers(), json: () => Promise.resolve({ choices: [{ text: expectedResponse }] }), }) const response = await adapter.getModelResponse(nonChatOptions) - expect(response).toBe(expectedResponse) + expect(response.prediction).toBe(expectedResponse) }) }) diff --git a/vscode/src/autoedits/adapters/fireworks.ts b/vscode/src/autoedits/adapters/fireworks.ts index f9da3d07318f..b412f6e53686 100644 --- a/vscode/src/autoedits/adapters/fireworks.ts +++ b/vscode/src/autoedits/adapters/fireworks.ts @@ -1,7 +1,7 @@ import { autoeditsProviderConfig } from '../autoedits-config' import { autoeditsOutputChannelLogger } from '../output-channel-logger' -import type { AutoeditModelOptions, AutoeditsModelAdapter } from './base' +import type { AutoeditModelOptions, AutoeditsModelAdapter, ModelResponse } from './base' import { type FireworksCompatibleRequestParams, getMaxOutputTokensForAutoedits, @@ -10,7 +10,7 @@ import { } from './utils' export class FireworksAdapter implements AutoeditsModelAdapter { - async getModelResponse(option: AutoeditModelOptions): Promise { + async getModelResponse(option: AutoeditModelOptions): Promise { const body = this.getMessageBody(option) try { const apiKey = autoeditsProviderConfig.experimentalAutoeditsConfigOverride?.apiKey @@ -22,11 +22,25 @@ export class FireworksAdapter implements AutoeditsModelAdapter { ) throw new Error('No api key provided in the config override') } - const response = await getModelResponse(option.url, body, apiKey) + const { data, requestHeaders, responseHeaders, url } = await getModelResponse( + option.url, + body, + apiKey + ) + + let prediction: string if (option.isChatModel) { - return response.choices[0].message.content + prediction = data.choices[0].message.content + } else { + prediction = data.choices[0].text + } + + return { + prediction, + responseHeaders, + requestHeaders, + requestUrl: url, } - return response.choices[0].text } catch (error) { autoeditsOutputChannelLogger.logError('getModelResponse', 'Error calling Fireworks API:', { verbose: error, diff --git a/vscode/src/autoedits/adapters/openai.ts b/vscode/src/autoedits/adapters/openai.ts index d8d9fa215b10..73a57138baaf 100644 --- a/vscode/src/autoedits/adapters/openai.ts +++ b/vscode/src/autoedits/adapters/openai.ts @@ -1,11 +1,11 @@ import { autoeditsProviderConfig } from '../autoedits-config' import { autoeditsOutputChannelLogger } from '../output-channel-logger' -import type { AutoeditModelOptions, AutoeditsModelAdapter } from './base' +import type { AutoeditModelOptions, AutoeditsModelAdapter, ModelResponse } from './base' import { getModelResponse, getOpenaiCompatibleChatPrompt } from './utils' export class OpenAIAdapter implements AutoeditsModelAdapter { - async getModelResponse(options: AutoeditModelOptions): Promise { + async getModelResponse(options: AutoeditModelOptions): Promise { try { const apiKey = autoeditsProviderConfig.experimentalAutoeditsConfigOverride?.apiKey @@ -17,7 +17,7 @@ export class OpenAIAdapter implements AutoeditsModelAdapter { throw new Error('No api key provided in the config override') } - const response = await getModelResponse( + const { data, requestHeaders, responseHeaders, url } = await getModelResponse( options.url, JSON.stringify({ model: options.model, @@ -33,7 +33,13 @@ export class OpenAIAdapter implements AutoeditsModelAdapter { }), apiKey ) - return response.choices[0].message.content + + return { + prediction: data.choices[0].message.content, + responseHeaders, + requestHeaders, + requestUrl: url, + } } catch (error) { autoeditsOutputChannelLogger.logError('getModelResponse', 'Error calling OpenAI API:', { verbose: error, diff --git a/vscode/src/autoedits/adapters/sourcegraph-chat.test.ts b/vscode/src/autoedits/adapters/sourcegraph-chat.test.ts index 23a4030b8c88..08dc9e6ec2c9 100644 --- a/vscode/src/autoedits/adapters/sourcegraph-chat.test.ts +++ b/vscode/src/autoedits/adapters/sourcegraph-chat.test.ts @@ -84,7 +84,7 @@ describe('SourcegraphChatAdapter', () => { mockChatClient.chat = mockChat const response = await adapter.getModelResponse(options) - expect(response).toBe('part1part2') + expect(response.prediction).toBe('part1part2') }) it('handles errors correctly', async () => { diff --git a/vscode/src/autoedits/adapters/sourcegraph-chat.ts b/vscode/src/autoedits/adapters/sourcegraph-chat.ts index 2d231aa20ad4..d72cca431571 100644 --- a/vscode/src/autoedits/adapters/sourcegraph-chat.ts +++ b/vscode/src/autoedits/adapters/sourcegraph-chat.ts @@ -1,12 +1,12 @@ import type { ChatClient, Message } from '@sourcegraph/cody-shared' import { autoeditsOutputChannelLogger } from '../output-channel-logger' -import type { AutoeditModelOptions, AutoeditsModelAdapter } from './base' +import type { AutoeditModelOptions, AutoeditsModelAdapter, ModelResponse } from './base' import { getMaxOutputTokensForAutoedits, getSourcegraphCompatibleChatPrompt } from './utils' export class SourcegraphChatAdapter implements AutoeditsModelAdapter { constructor(private readonly chatClient: ChatClient) {} - async getModelResponse(option: AutoeditModelOptions): Promise { + async getModelResponse(option: AutoeditModelOptions): Promise { try { const maxTokens = getMaxOutputTokensForAutoedits(option.codeToRewrite) const messages: Message[] = getSourcegraphCompatibleChatPrompt({ @@ -36,7 +36,13 @@ export class SourcegraphChatAdapter implements AutoeditsModelAdapter { break } } - return accumulated + + // For direct API calls without HTTP headers, we return an empty object + return { + prediction: accumulated, + responseHeaders: {}, + requestUrl: option.url, + } } catch (error) { autoeditsOutputChannelLogger.logError( 'getModelResponse', diff --git a/vscode/src/autoedits/adapters/sourcegraph-completions.test.ts b/vscode/src/autoedits/adapters/sourcegraph-completions.test.ts index 1a37a4b7d525..f3b5dcef884b 100644 --- a/vscode/src/autoedits/adapters/sourcegraph-completions.test.ts +++ b/vscode/src/autoedits/adapters/sourcegraph-completions.test.ts @@ -79,7 +79,7 @@ describe('SourcegraphCompletionsAdapter', () => { adapter.client = { complete: mockComplete } const response = await adapter.getModelResponse(options) - expect(response).toBe('part1part2') + expect(response.prediction).toBe('part1part2') }) it('handles errors correctly', async () => { diff --git a/vscode/src/autoedits/adapters/sourcegraph-completions.ts b/vscode/src/autoedits/adapters/sourcegraph-completions.ts index ffd09d92a978..23975c89b294 100644 --- a/vscode/src/autoedits/adapters/sourcegraph-completions.ts +++ b/vscode/src/autoedits/adapters/sourcegraph-completions.ts @@ -6,7 +6,7 @@ import type { } from '@sourcegraph/cody-shared' import { defaultCodeCompletionsClient } from '../../completions/default-client' import { autoeditsOutputChannelLogger } from '../output-channel-logger' -import type { AutoeditModelOptions, AutoeditsModelAdapter } from './base' +import type { AutoeditModelOptions, AutoeditsModelAdapter, ModelResponse } from './base' import { getMaxOutputTokensForAutoedits, getSourcegraphCompatibleChatPrompt } from './utils' export class SourcegraphCompletionsAdapter implements AutoeditsModelAdapter { @@ -16,14 +16,14 @@ export class SourcegraphCompletionsAdapter implements AutoeditsModelAdapter { this.client = defaultCodeCompletionsClient.instance! } - async getModelResponse(option: AutoeditModelOptions): Promise { + async getModelResponse(option: AutoeditModelOptions): Promise { try { const maxTokens = getMaxOutputTokensForAutoedits(option.codeToRewrite) const messages: Message[] = getSourcegraphCompatibleChatPrompt({ - systemMessage: undefined, + systemMessage: option.prompt.systemMessage, userMessage: option.prompt.userMessage, }) - const requestParam: CodeCompletionsParams = { + const requestBody: CodeCompletionsParams = { timeoutMs: 5_000, model: option.model as ModelRefStr, messages, @@ -34,19 +34,58 @@ export class SourcegraphCompletionsAdapter implements AutoeditsModelAdapter { content: option.codeToRewrite, }, } - const completionResponseGenerator = await this.client.complete( - requestParam, - new AbortController() - ) - let accumulated = '' + // Create an AbortController to pass to the client + const abortController = new AbortController() + + const completionResponseGenerator = await this.client.complete(requestBody, abortController) + + let prediction = '' + let responseBody: any = null + let responseHeaders: Record = {} + let requestHeaders: Record = {} + let requestUrl = option.url + for await (const msg of completionResponseGenerator) { const newText = msg.completionResponse?.completion if (newText) { - accumulated = newText + prediction = newText } + + // Capture response metadata if available + if (msg.metadata) { + if (msg.metadata.response) { + // Extract headers into a plain object + responseHeaders = {} + msg.metadata.response.headers.forEach((value, key) => { + responseHeaders[key] = value + }) + } + + // Capture request metadata + if (msg.metadata.requestHeaders) { + requestHeaders = msg.metadata.requestHeaders + } + + if (msg.metadata.requestUrl) { + requestUrl = msg.metadata.requestUrl + } + + // Store the full response body if available + if (msg.completionResponse) { + responseBody = msg.completionResponse + } + } + } + + return { + prediction, + responseHeaders, + requestHeaders, + requestUrl, + requestBody, + responseBody, } - return accumulated } catch (error) { autoeditsOutputChannelLogger.logError( 'getModelResponse', diff --git a/vscode/src/autoedits/adapters/utils.ts b/vscode/src/autoedits/adapters/utils.ts index 59de87d4b8fc..7e46f8fc900d 100644 --- a/vscode/src/autoedits/adapters/utils.ts +++ b/vscode/src/autoedits/adapters/utils.ts @@ -16,6 +16,23 @@ export interface FireworksCompatibleRequestParams { user?: string } +export interface FireworksChatMessage { + role: string + content: PromptString +} + +export interface FireworksChatModelRequestParams extends FireworksCompatibleRequestParams { + messages: FireworksChatMessage[] +} + +export interface FireworksCompletionModelRequestParams extends FireworksCompatibleRequestParams { + prompt: PromptString +} + +export type AutoeditsRequestBody = + | FireworksChatModelRequestParams + | FireworksCompletionModelRequestParams + export function getMaxOutputTokensForAutoedits(codeToRewrite: string): number { const MAX_NEW_GENERATED_TOKENS = 512 const codeToRewriteTokens = charsToTokens(codeToRewrite.length) @@ -51,20 +68,33 @@ export async function getModelResponse( body: string, apiKey: string, customHeaders: Record = {} -): Promise { +): Promise<{ + data: any + requestHeaders: Record + responseHeaders: Record + url: string +}> { + const requestHeaders = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + ...customHeaders, + } const response = await fetch(url, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, - ...customHeaders, - }, + headers: requestHeaders, body: body, }) if (response.status !== 200) { const errorText = await response.text() throw new Error(`HTTP error! status: ${response.status}, message: ${errorText}`) } + + // Extract headers into a plain object + const responseHeaders: Record = {} + response.headers.forEach((value, key) => { + responseHeaders[key] = value + }) + const data = await response.json() - return data + return { data, requestHeaders, responseHeaders, url } } diff --git a/vscode/src/autoedits/autoedits-provider.test.ts b/vscode/src/autoedits/autoedits-provider.test.ts index c438f788cd4d..44827b956d91 100644 --- a/vscode/src/autoedits/autoedits-provider.test.ts +++ b/vscode/src/autoedits/autoedits-provider.test.ts @@ -465,7 +465,14 @@ describe('AutoeditsProvider', () => { const customGetModelResponse = async () => { // Record the current fake timer time when getModelResponse is called getModelResponseCalledAt = Date.now() - return { choices: [{ text: 'const x = 1\n' }] } + return { + data: { + choices: [{ text: 'const x = 1\n' }], + }, + url: 'test-url.com/completions', + requestHeaders: {}, + responseHeaders: {}, + } } const startTime = Date.now() @@ -489,7 +496,12 @@ describe('AutoeditsProvider', () => { let modelResponseCalled = false const customGetModelResponse = async () => { modelResponseCalled = true - return { choices: [{ text: 'const x = 1\n' }] } + return { + data: { choices: [{ text: 'const x = 1\n' }] }, + url: 'test-url.com/completions', + requestHeaders: {}, + responseHeaders: {}, + } } const tokenSource = new vscode.CancellationTokenSource() diff --git a/vscode/src/autoedits/autoedits-provider.ts b/vscode/src/autoedits/autoedits-provider.ts index 780ca0f59369..fba6d924ebcc 100644 --- a/vscode/src/autoedits/autoedits-provider.ts +++ b/vscode/src/autoedits/autoedits-provider.ts @@ -12,7 +12,7 @@ import { isRunningInsideAgent } from '../jsonrpc/isRunningInsideAgent' import type { FixupController } from '../non-stop/FixupController' import type { CodyStatusBar } from '../services/StatusBar' -import type { AutoeditsModelAdapter, AutoeditsPrompt } from './adapters/base' +import type { AutoeditsModelAdapter, AutoeditsPrompt, ModelResponse } from './adapters/base' import { createAutoeditsModelAdapter } from './adapters/create-adapter' import { type AutoeditRequestID, @@ -258,7 +258,7 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v 'provideInlineCompletionItems', 'Calculating prediction from getPrediction...' ) - const initialPrediction = await this.getPrediction({ + const predictionResult = await this.getPrediction({ document, position, prompt, @@ -278,7 +278,7 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v return null } - if (initialPrediction === undefined || initialPrediction.length === 0) { + if (!predictionResult || predictionResult.prediction.length === 0) { autoeditsOutputChannelLogger.logDebugIfVerbose( 'provideInlineCompletionItems', 'received empty prediction' @@ -291,13 +291,15 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v return null } + const initialPrediction = predictionResult.prediction + autoeditAnalyticsLogger.markAsLoaded({ requestId, prompt, payload: { source: autoeditSource.network, isFuzzyMatch: false, - responseHeaders: {}, + responseHeaders: predictionResult.responseHeaders, prediction: initialPrediction, }, }) @@ -440,6 +442,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v ? error : new Error(`provideInlineCompletionItems autoedit error: ${error}`) + if (process.env.NODE_ENV === 'development') { + console.error(errorToReport) + } + autoeditAnalyticsLogger.logError(errorToReport) return null } finally { @@ -466,7 +472,7 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v position: vscode.Position codeToReplaceData: CodeToReplaceData prompt: AutoeditsPrompt - }): Promise { + }): Promise { if (autoeditsProviderConfig.isMockResponseFromCurrentDocumentTemplateEnabled) { const responseMetadata = extractAutoEditResponseFromCurrentDocumentCommentTemplate( document, @@ -480,7 +486,11 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v ) if (prediction) { - return prediction + return { + prediction, + responseHeaders: {}, + requestUrl: autoeditsProviderConfig.url, + } } } } diff --git a/vscode/src/autoedits/test-helpers.ts b/vscode/src/autoedits/test-helpers.ts index 14d4ce01b566..595cd5b799a2 100644 --- a/vscode/src/autoedits/test-helpers.ts +++ b/vscode/src/autoedits/test-helpers.ts @@ -47,7 +47,12 @@ export async function autoeditResultFor( body: string, apiKey: string, customHeaders?: Record - ) => Promise + ) => Promise<{ + data: any + requestHeaders: Record + responseHeaders: Record + url: string + }> isAutomaticTimersAdvancementDisabled?: boolean } ): Promise<{ @@ -63,11 +68,16 @@ export async function autoeditResultFor( vi.advanceTimersByTime(100) return { - choices: [ - { - text: prediction, - }, - ], + data: { + choices: [ + { + text: prediction, + }, + ], + }, + requestHeaders: {}, + responseHeaders: {}, + url: 'test-url.com/completions', } } diff --git a/vscode/src/completions/default-client.ts b/vscode/src/completions/default-client.ts index 9e1b6e4ac905..068cb4da8fa1 100644 --- a/vscode/src/completions/default-client.ts +++ b/vscode/src/completions/default-client.ts @@ -87,6 +87,12 @@ class DefaultCodeCompletionsClient implements CodeCompletionsClient { throw recordErrorToSpan(span, error) } + // Convert Headers to Record for requestHeaders + const requestHeaders: Record = {} + headers.forEach((value, key) => { + requestHeaders[key] = value + }) + // We enable streaming only for Node environments right now because it's hard to make // the polyfilled fetch API work the same as it does in the browser. // @@ -165,7 +171,12 @@ class DefaultCodeCompletionsClient implements CodeCompletionsClient { const result: CompletionResponseWithMetaData = { completionResponse: undefined, - metadata: { response }, + metadata: { + response, + requestHeaders, + requestUrl: url.toString(), + requestBody: serializedParams, + }, } try { From d6f01ab3d79a56e2239b84b7e2228fc56951ac3b Mon Sep 17 00:00:00 2001 From: Valery Bugakov Date: Tue, 4 Mar 2025 11:39:55 +0800 Subject: [PATCH 2/4] chore(auto-edit): integrate model response metadata with analytics logger --- .../analytics-logger/analytics-logger.test.ts | 12 +- .../analytics-logger/analytics-logger.ts | 372 ++---------------- .../src/autoedits/analytics-logger/index.ts | 1 + .../suggestion-id-registry.ts | 2 +- .../src/autoedits/analytics-logger/types.ts | 341 ++++++++++++++++ vscode/src/autoedits/autoedits-provider.ts | 2 +- 6 files changed, 388 insertions(+), 342 deletions(-) create mode 100644 vscode/src/autoedits/analytics-logger/types.ts diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts index b45441a12cc6..07fe1e632025 100644 --- a/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts +++ b/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts @@ -17,13 +17,13 @@ import type { AutoeditModelOptions } from '../adapters/base' import { getCodeToReplaceData } from '../prompt/prompt-utils' import { getDecorationInfo } from '../renderer/diff-utils' +import { AutoeditAnalyticsLogger } from './analytics-logger' import { - AutoeditAnalyticsLogger, type AutoeditRequestID, autoeditDiscardReason, autoeditSource, autoeditTriggerKind, -} from './analytics-logger' +} from './types' describe('AutoeditAnalyticsLogger', () => { let autoeditLogger: AutoeditAnalyticsLogger @@ -109,11 +109,17 @@ describe('AutoeditAnalyticsLogger', () => { autoeditLogger.markAsLoaded({ requestId, prompt: modelOptions.prompt, + modelResponse: { + prediction, + requestHeaders: {}, + requestUrl: modelOptions.url, + responseHeaders: {}, + responseBody: {}, + }, payload: { prediction, source: autoeditSource.network, isFuzzyMatch: false, - responseHeaders: {}, }, }) diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.ts index 404150e54987..a3ee48a38a50 100644 --- a/vscode/src/autoedits/analytics-logger/analytics-logger.ts +++ b/vscode/src/autoedits/analytics-logger/analytics-logger.ts @@ -14,343 +14,32 @@ import { import type { TelemetryEventParameters } from '@sourcegraph/telemetry' import { getOtherCompletionProvider } from '../../completions/analytics-logger' -import type { ContextSummary } from '../../completions/context/context-mixer' import { lines } from '../../completions/text-processing' -import { type CodeGenEventMetadata, charactersLogger } from '../../services/CharactersLogger' +import { charactersLogger } from '../../services/CharactersLogger' import { upstreamHealthProvider } from '../../services/UpstreamHealthProvider' import { captureException, shouldErrorBeReported } from '../../services/sentry/sentry' import { splitSafeMetadata } from '../../services/telemetry-v2' -import type { AutoeditsPrompt } from '../adapters/base' +import type { AutoeditsPrompt, ModelResponse } from '../adapters/base' import { autoeditsOutputChannelLogger } from '../output-channel-logger' import type { CodeToReplaceData } from '../prompt/prompt-utils' import type { DecorationInfo } from '../renderer/decorators/base' -import { type DecorationStats, getDecorationStats } from '../renderer/diff-utils' +import { getDecorationStats } from '../renderer/diff-utils' import { autoeditIdRegistry } from './suggestion-id-registry' - -/** - * This file implements a state machine to manage the lifecycle of an autoedit request. - * Each phase of the request is represented by a distinct state interface, and metadata - * evolves as the request progresses. - * - * 1. Each autoedit request phase (e.g., `started`, `loaded`, `accepted`) is represented by a - * `state` interface that extends `AutoeditBaseState` and adds phase-specific fields. - * - * 2. Valid transitions between phases are enforced using the `validRequestTransitions` map, - * ensuring logical progression through the request lifecycle. - * - * 3. The `payload` field in each state encapsulates the exact list of fields that we plan to send - * to our analytics backend. - * - * 4. Other top-level `state` fields are saved only for bookkeeping and won't end up at our - * analytics backend. This ensures we don't send unintentional or redundant information to - * the analytics backend. - * - * 5. Metadata is progressively enriched as the request transitions between states. - * - * 6. Eventually, once we reach one of the terminal states and log its current `payload`. - */ - -/** - * Defines the possible phases of our autoedit request state machine. - */ -type Phase = - /** The autoedit request has started. */ - | 'started' - /** The context for the autoedit has been loaded. */ - | 'contextLoaded' - /** The autoedit suggestion has been loaded — we have a prediction string. */ - | 'loaded' - /** - * The suggestion is not discard during post processing and we have all the data to render the suggestion. - * This intermediate step is required for the agent API. We cannot graduate the request to the suggested - * state right away. We first need to save requests metadata to the analytics logger cache, so that - * agent can access it using the request ID only in `unstable_handleDidShowCompletionItem` calls. - */ - | 'postProcessed' - /** The autoedit suggestion has been suggested to the user. */ - | 'suggested' - /** The autoedit suggestion is marked as read is it's still visible to the user after a hardcoded timeout. */ - | 'read' - /** The user has accepted the suggestion. */ - | 'accepted' - /** The user has rejected the suggestion. */ - | 'rejected' - /** The autoedit request was discarded by our heuristics before being suggested to a user */ - | 'discarded' - -/** - * Defines which phases can transition to which other phases. - */ -const validRequestTransitions = { - started: ['contextLoaded', 'discarded'], - contextLoaded: ['loaded', 'discarded'], - loaded: ['postProcessed', 'discarded'], - postProcessed: ['suggested', 'discarded'], - suggested: ['read', 'accepted', 'rejected'], - read: ['accepted', 'rejected'], - accepted: [], - rejected: [], - discarded: [], -} as const satisfies Record - -export const autoeditTriggerKind = { - /** Suggestion was triggered automatically while editing. */ - automatic: 1, - - /** Suggestion was triggered manually by the user invoking the keyboard shortcut. */ - manual: 2, - - /** When the user uses the suggest widget to cycle through different suggestions. */ - suggestWidget: 3, - - /** Suggestion was triggered automatically by the selection change event. */ - cursor: 4, -} as const - -/** We use numeric keys to send these to the analytics backend */ -type AutoeditTriggerKindMetadata = (typeof autoeditTriggerKind)[keyof typeof autoeditTriggerKind] - -interface AutoeditStartedMetadata { - /** Document language ID (e.g., 'typescript'). */ - languageId: string - - /** Model used by Cody client to request the autosuggestion suggestion. */ - model: string - - /** Optional trace ID for cross-service correlation, if your environment provides it. */ - traceId?: string - - /** Describes how the autoedit request was triggered by the user. */ - triggerKind: AutoeditTriggerKindMetadata - - /** - * The code to rewrite by autoedit. - * 🚨 SECURITY: included only for DotCom users. - */ - codeToRewrite?: string - - /** True if other autoedit/completion providers might also be active (e.g., Copilot). */ - otherCompletionProviderEnabled: boolean - - /** The exact list of other providers that are active, if known. */ - otherCompletionProviders: string[] - - /** The round trip timings to reach the Sourcegraph and Cody Gateway instances. */ - upstreamLatency?: number - gatewayLatency?: number -} - -interface AutoeditContextLoadedMetadata extends AutoeditStartedMetadata { - /** - * Information about the context retrieval process that lead to this autoedit request. Refer - * to the documentation of {@link ContextSummary} - */ - contextSummary?: ContextSummary -} - -/** - * A stable ID that identifies a particular autoedit suggestion. If the same text - * and context recurs, we reuse this ID to avoid double-counting. - */ -export type AutoeditSuggestionID = string & { readonly _brand: 'AutoeditSuggestionID' } - -export const autoeditSource = { - /** Autoedit originated from a request to our backend for the suggestion. */ - network: 1, - /** Autoedit originated from a client cached suggestion. */ - cache: 2, -} as const - -/** We use numeric keys to send these to the analytics backend */ -type AutoeditSourceMetadata = (typeof autoeditSource)[keyof typeof autoeditSource] - -export const autoeditDiscardReason = { - clientAborted: 1, - emptyPrediction: 2, - predictionEqualsCodeToRewrite: 3, - recentEdits: 4, - suffixOverlap: 5, - emptyPredictionAfterInlineCompletionExtraction: 6, - noActiveEditor: 7, - conflictingDecorationWithEdits: 8, - notEnoughLinesEditor: 9, -} as const - -/** We use numeric keys to send these to the analytics backend */ -type AutoeditDiscardReasonMetadata = (typeof autoeditDiscardReason)[keyof typeof autoeditDiscardReason] - -interface AutoeditLoadedMetadata extends AutoeditContextLoadedMetadata { - /** - * An ID to uniquely identify a suggest autoedit. Note: It is possible for this ID to be part - * of two suggested events. This happens when the exact same autoedit text is shown again at - * the exact same location. We count this as the same autoedit and thus use the same ID. - */ - id: AutoeditSuggestionID - - /** - * Unmodified by the client prediction text snippet of the suggestion. - * Might be `undefined` if too long. - * 🚨 SECURITY: included only for DotCom users. - */ - prediction?: string - - /** The source of the suggestion, e.g. 'network', 'cache', etc. */ - source?: AutoeditSourceMetadata - - /** True if we fuzzy-matched this suggestion from a local or remote cache. */ - isFuzzyMatch?: boolean - - /** Optional set of relevant response headers (e.g. from Cody Gateway). */ - responseHeaders?: Record - - /** Time (ms) to generate or load the suggestion after it was started. */ - latency: number -} - -interface AutoeditPostProcessedMetadata extends AutoeditLoadedMetadata { - /** The number of added, modified, removed lines and characters from suggestion. */ - decorationStats?: DecorationStats - /** The number of lines and added chars attributed to an inline completion item. */ - inlineCompletionStats?: { - lineCount: number - charCount: number - } -} - -interface AutoEditFinalMetadata extends AutoeditPostProcessedMetadata { - /** Displayed to the user for this many milliseconds. */ - timeFromSuggestedAt: number - /** True if the suggestion was explicitly/intentionally accepted. */ - isAccepted: boolean - /** - * True if the suggestion was visible for a certain time - * Required to correctly calculate CAR and other metrics where we - * want to account only for suggestions visible for a certain time. - * - * `timeFromSuggestedAt` is not a reliable source of truth for - * this case because a user could have rejected a suggestion without - * triggering `accepted` or `discarded` immediately. This is related to - * limited VS Code APIs which do not provide a reliable way to know - * if a suggestion is really visible. - */ - isRead: boolean - /** The number of the auto-edit started since the last suggestion was shown. */ - suggestionsStartedSinceLastSuggestion: number -} - -interface AutoeditAcceptedEventPayload - extends AutoEditFinalMetadata, - Omit {} - -interface AutoeditRejectedEventPayload extends AutoEditFinalMetadata {} -interface AutoeditDiscardedEventPayload extends AutoeditContextLoadedMetadata { - discardReason: AutoeditDiscardReasonMetadata -} - -/** - * An ephemeral ID for a single “request” from creation to acceptance or rejection. - */ -export type AutoeditRequestID = string & { readonly _brand: 'AutoeditRequestID' } - -/** - * The base fields common to all request states. We track ephemeral times and - * the partial payload. Once we reach a certain phase, we log the payload as a telemetry event. - */ -interface AutoeditBaseState { - requestId: AutoeditRequestID - /** Current phase of the autoedit request */ - phase: Phase -} - -interface StartedState extends AutoeditBaseState { - phase: 'started' - /** Time (ms) when we started computing or requesting the suggestion. */ - startedAt: number - - /** Metadata required to show a suggestion based on `requestId` only. */ - codeToReplaceData: CodeToReplaceData - document: vscode.TextDocument - position: vscode.Position - docContext: DocumentContext - - /** Partial payload for this phase. Will be augmented with more info as we progress. */ - payload: AutoeditStartedMetadata -} - -export interface ContextLoadedState extends Omit { - phase: 'contextLoaded' - payload: AutoeditContextLoadedMetadata -} - -interface LoadedState extends Omit { - phase: 'loaded' - /** Timestamp when the suggestion completed generation/loading. */ - loadedAt: number - payload: AutoeditLoadedMetadata -} - -export interface PostProcessedState extends Omit { - phase: 'postProcessed' - - /** Metadata required to show a suggestion based on `requestId` only. */ - prediction: string - decorationInfo: DecorationInfo | null - inlineCompletionItems: vscode.InlineCompletionItem[] | null - - payload: AutoeditPostProcessedMetadata -} - -export interface SuggestedState extends Omit { - phase: 'suggested' - /** Timestamp when the suggestion was first shown to the user. */ - suggestedAt: number -} - -export interface ReadState extends Omit { - phase: 'read' - /** Timestamp when the suggestion was marked as visible to the user. */ - readAt: number -} - -export interface AcceptedState extends Omit { - phase: 'accepted' - /** Timestamp when the user accepted the suggestion. */ - acceptedAt: number - /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ - suggestionLoggedAt?: number - /** Optional because it might be accepted before the read timeout */ - readAt?: number - payload: AutoeditAcceptedEventPayload -} - -export interface RejectedState extends Omit { - phase: 'rejected' - /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ - suggestionLoggedAt?: number - /** Optional because it might be accepted before the read timeout */ - readAt?: number - payload: AutoeditRejectedEventPayload -} - -interface DiscardedState extends Omit { - phase: 'discarded' - /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ - suggestionLoggedAt?: number - payload: AutoeditDiscardedEventPayload -} - -interface PhaseStates { - started: StartedState - contextLoaded: ContextLoadedState - loaded: LoadedState - postProcessed: PostProcessedState - suggested: SuggestedState - read: ReadState - accepted: AcceptedState - rejected: RejectedState - discarded: DiscardedState -} +import { + type AcceptedState, + type AutoeditDiscardReasonMetadata, + type AutoeditRequestID, + type ContextLoadedState, + type DiscardedState, + type LoadedState, + type Phase, + type PhaseStates, + type RejectedState, + type StartedState, + type SuggestedState, + validRequestTransitions, +} from './types' /** * Using the validTransitions definition, we can derive which "from phases" lead to a given next phase, @@ -410,7 +99,7 @@ export class AutoeditAnalyticsLogger { position: vscode.Position docContext: DocumentContext payload: Required< - Pick + Pick > }): AutoeditRequestID { const { codeToRewrite, ...restPayload } = payload @@ -438,6 +127,7 @@ export class AutoeditAnalyticsLogger { this.activeRequests.set(requestId, request) this.autoeditsStartedSinceLastSuggestion++ + return requestId } @@ -446,10 +136,11 @@ export class AutoeditAnalyticsLogger { payload, }: { requestId: AutoeditRequestID - payload: Pick + payload: Pick }): void { this.tryTransitionTo(requestId, 'contextLoaded', request => ({ ...request, + contextLoadedAt: getTimeNowInMillis(), payload: { ...request.payload, contextSummary: payload.contextSummary, @@ -466,14 +157,14 @@ export class AutoeditAnalyticsLogger { requestId, prompt, payload, + modelResponse, }: { + modelResponse: ModelResponse requestId: AutoeditRequestID prompt: AutoeditsPrompt - payload: Required< - Pick - > + payload: Required> }): void { - const { prediction, source, isFuzzyMatch, responseHeaders } = payload + const { prediction, source, isFuzzyMatch } = payload const stableId = autoeditIdRegistry.getOrCreate(prompt, prediction) const loadedAt = getTimeNowInMillis() @@ -481,6 +172,7 @@ export class AutoeditAnalyticsLogger { return { ...request, loadedAt, + modelResponse, payload: { ...request.payload, id: stableId, @@ -488,7 +180,7 @@ export class AutoeditAnalyticsLogger { prediction: isDotComAuthed() && prediction.length < 300 ? prediction : undefined, source, isFuzzyMatch, - responseHeaders, + responseHeaders: modelResponse.responseHeaders, latency: Math.floor(loadedAt - request.startedAt), }, } @@ -515,6 +207,7 @@ export class AutoeditAnalyticsLogger { return { ...request, + postProcessedAt: getTimeNowInMillis(), prediction, decorationInfo, inlineCompletionItems, @@ -598,13 +291,16 @@ export class AutoeditAnalyticsLogger { } public markAsRejected(requestId: AutoeditRequestID): void { + const rejectedAt = getTimeNowInMillis() + const result = this.tryTransitionTo(requestId, 'rejected', request => ({ ...request, + rejectedAt, payload: { ...request.payload, isAccepted: false, isRead: 'readAt' in request, - timeFromSuggestedAt: getTimeNowInMillis() - request.suggestedAt, + timeFromSuggestedAt: rejectedAt - request.suggestedAt, suggestionsStartedSinceLastSuggestion: this.autoeditsStartedSinceLastSuggestion, }, })) @@ -628,9 +324,10 @@ export class AutoeditAnalyticsLogger { const result = this.tryTransitionTo(requestId, 'discarded', request => { return { ...request, + discardedAt: getTimeNowInMillis(), payload: { ...request.payload, - discardReason: discardReason, + discardReason, }, } }) @@ -662,6 +359,7 @@ export class AutoeditAnalyticsLogger { } as PhaseStates[P] this.activeRequests.set(requestId, updatedRequest) + return { updatedRequest, currentRequest } } diff --git a/vscode/src/autoedits/analytics-logger/index.ts b/vscode/src/autoedits/analytics-logger/index.ts index 6ae02d5165b1..410caf032de7 100644 --- a/vscode/src/autoedits/analytics-logger/index.ts +++ b/vscode/src/autoedits/analytics-logger/index.ts @@ -1 +1,2 @@ export * from './analytics-logger' +export * from './types' diff --git a/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts b/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts index 11cef9e27251..043718bae507 100644 --- a/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts +++ b/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts @@ -2,7 +2,7 @@ import { LRUCache } from 'lru-cache' import * as uuid from 'uuid' import type { AutoeditsPrompt } from '../adapters/base' -import type { AutoeditSuggestionID } from './analytics-logger' +import type { AutoeditSuggestionID } from './types' /** * A specialized string type for the stable “suggestion key” in caches. diff --git a/vscode/src/autoedits/analytics-logger/types.ts b/vscode/src/autoedits/analytics-logger/types.ts new file mode 100644 index 000000000000..7f6f86870362 --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/types.ts @@ -0,0 +1,341 @@ +import type * as vscode from 'vscode' + +import type { DocumentContext } from '@sourcegraph/cody-shared' + +import type { ContextSummary } from '../../completions/context/context-mixer' +import type { CodeGenEventMetadata } from '../../services/CharactersLogger' +import type { ModelResponse } from '../adapters/base' +import type { CodeToReplaceData } from '../prompt/prompt-utils' +import type { DecorationInfo } from '../renderer/decorators/base' +import type { DecorationStats } from '../renderer/diff-utils' + +/** + * This file implements a state machine to manage the lifecycle of an autoedit request. + * Each phase of the request is represented by a distinct state interface, and metadata + * evolves as the request progresses. + * + * 1. Each autoedit request phase (e.g., `started`, `loaded`, `accepted`) is represented by a + * `state` interface that extends `AutoeditBaseState` and adds phase-specific fields. + * + * 2. Valid transitions between phases are enforced using the `validRequestTransitions` map, + * ensuring logical progression through the request lifecycle. + * + * 3. The `payload` field in each state encapsulates the exact list of fields that we plan to send + * to our analytics backend. + * + * 4. Other top-level `state` fields are saved only for bookkeeping and won't end up at our + * analytics backend. This ensures we don't send unintentional or redundant information to + * the analytics backend. + * + * 5. Metadata is progressively enriched as the request transitions between states. + * + * 6. Eventually, once we reach one of the terminal states and log its current `payload`. + */ + +/** + * Defines the possible phases of our autoedit request state machine. + */ +export type Phase = + /** The autoedit request has started. */ + | 'started' + /** The context for the autoedit has been loaded. */ + | 'contextLoaded' + /** The autoedit suggestion has been loaded — we have a prediction string. */ + | 'loaded' + /** + * The suggestion is not discard during post processing and we have all the data to render the suggestion. + * This intermediate step is required for the agent API. We cannot graduate the request to the suggested + * state right away. We first need to save requests metadata to the analytics logger cache, so that + * agent can access it using the request ID only in `unstable_handleDidShowCompletionItem` calls. + */ + | 'postProcessed' + /** The autoedit suggestion has been suggested to the user. */ + | 'suggested' + /** The autoedit suggestion is marked as read is it's still visible to the user after a hardcoded timeout. */ + | 'read' + /** The user has accepted the suggestion. */ + | 'accepted' + /** The user has rejected the suggestion. */ + | 'rejected' + /** The autoedit request was discarded by our heuristics before being suggested to a user */ + | 'discarded' + +/** + * Defines which phases can transition to which other phases. + */ +export const validRequestTransitions = { + started: ['contextLoaded', 'discarded'], + contextLoaded: ['loaded', 'discarded'], + loaded: ['postProcessed', 'discarded'], + postProcessed: ['suggested', 'discarded'], + suggested: ['read', 'accepted', 'rejected'], + read: ['accepted', 'rejected'], + accepted: [], + rejected: [], + discarded: [], +} as const satisfies Record + +export const autoeditTriggerKind = { + /** Suggestion was triggered automatically while editing. */ + automatic: 1, + + /** Suggestion was triggered manually by the user invoking the keyboard shortcut. */ + manual: 2, + + /** When the user uses the suggest widget to cycle through different suggestions. */ + suggestWidget: 3, + + /** Suggestion was triggered automatically by the selection change event. */ + cursor: 4, +} as const + +/** We use numeric keys to send these to the analytics backend */ +export type AutoeditTriggerKindMetadata = (typeof autoeditTriggerKind)[keyof typeof autoeditTriggerKind] + +export const autoeditSource = { + /** Autoedit originated from a request to our backend for the suggestion. */ + network: 1, + /** Autoedit originated from a client cached suggestion. */ + cache: 2, +} as const + +/** We use numeric keys to send these to the analytics backend */ +export type AutoeditSourceMetadata = (typeof autoeditSource)[keyof typeof autoeditSource] + +export const autoeditDiscardReason = { + clientAborted: 1, + emptyPrediction: 2, + predictionEqualsCodeToRewrite: 3, + recentEdits: 4, + suffixOverlap: 5, + emptyPredictionAfterInlineCompletionExtraction: 6, + noActiveEditor: 7, + conflictingDecorationWithEdits: 8, + notEnoughLinesEditor: 9, +} as const + +/** We use numeric keys to send these to the analytics backend */ +export type AutoeditDiscardReasonMetadata = + (typeof autoeditDiscardReason)[keyof typeof autoeditDiscardReason] + +/** + * A stable ID that identifies a particular autoedit suggestion. If the same text + * and context recurs, we reuse this ID to avoid double-counting. + */ +export type AutoeditSuggestionID = string & { readonly _brand: 'AutoeditSuggestionID' } + +/** + * An ephemeral ID for a single "request" from creation to acceptance or rejection. + */ +export type AutoeditRequestID = string & { readonly _brand: 'AutoeditRequestID' } + +/** + * The base fields common to all request states. We track ephemeral times and + * the partial payload. Once we reach a certain phase, we log the payload as a telemetry event. + */ +export interface AutoeditBaseState { + requestId: AutoeditRequestID + /** Current phase of the autoedit request */ + phase: Phase +} + +export interface StartedState extends AutoeditBaseState { + phase: 'started' + /** Time (ms) when we started computing or requesting the suggestion. */ + startedAt: number + + /** Metadata required to show a suggestion based on `requestId` only. */ + codeToReplaceData: CodeToReplaceData + document: vscode.TextDocument + position: vscode.Position + docContext: DocumentContext + + /** Partial payload for this phase. Will be augmented with more info as we progress. */ + payload: { + /** Document language ID (e.g., 'typescript'). */ + languageId: string + + /** Model used by Cody client to request the autosuggestion suggestion. */ + model: string + + /** Optional trace ID for cross-service correlation, if your environment provides it. */ + traceId?: string + + /** Describes how the autoedit request was triggered by the user. */ + triggerKind: AutoeditTriggerKindMetadata + + /** + * The code to rewrite by autoedit. + * 🚨 SECURITY: included only for DotCom users. + */ + codeToRewrite?: string + + /** True if other autoedit/completion providers might also be active (e.g., Copilot). */ + otherCompletionProviderEnabled: boolean + + /** The exact list of other providers that are active, if known. */ + otherCompletionProviders: string[] + + /** The round trip timings to reach the Sourcegraph and Cody Gateway instances. */ + upstreamLatency?: number + gatewayLatency?: number + } +} + +export interface ContextLoadedState extends Omit { + phase: 'contextLoaded' + /** Timestamp when the context for the autoedit was loaded. */ + contextLoadedAt: number + payload: StartedState['payload'] & { + /** + * Information about the context retrieval process that lead to this autoedit request. Refer + * to the documentation of {@link ContextSummary} + */ + contextSummary?: ContextSummary + } +} + +export interface LoadedState extends Omit { + phase: 'loaded' + /** Timestamp when the suggestion completed generation/loading. */ + loadedAt: number + /** Model response metadata for the debug panel */ + modelResponse: ModelResponse + payload: ContextLoadedState['payload'] & { + /** + * An ID to uniquely identify a suggest autoedit. Note: It is possible for this ID to be part + * of two suggested events. This happens when the exact same autoedit text is shown again at + * the exact same location. We count this as the same autoedit and thus use the same ID. + */ + id: AutoeditSuggestionID + + /** + * Unmodified by the client prediction text snippet of the suggestion. + * Might be `undefined` if too long. + * 🚨 SECURITY: included only for DotCom users. + */ + prediction?: string + + /** The source of the suggestion, e.g. 'network', 'cache', etc. */ + source?: AutoeditSourceMetadata + + /** True if we fuzzy-matched this suggestion from a local or remote cache. */ + isFuzzyMatch?: boolean + + /** Optional set of relevant response headers (e.g. from Cody Gateway). */ + responseHeaders?: Record + + /** Time (ms) to generate or load the suggestion after it was started. */ + latency: number + } +} + +export interface PostProcessedState extends Omit { + phase: 'postProcessed' + /** Timestamp when the post-processing of the suggestion was completed. */ + postProcessedAt: number + + /** Metadata required to show a suggestion based on `requestId` only. */ + prediction: string + /** + * The decoration info after the post-processing of the suggestion. + * Won't include insertions rendered as inline completions. + */ + decorationInfo: DecorationInfo | null + inlineCompletionItems: vscode.InlineCompletionItem[] | null + + payload: LoadedState['payload'] & { + /** The number of added, modified, removed lines and characters from suggestion. */ + decorationStats?: DecorationStats + /** The number of lines and added chars attributed to an inline completion item. */ + inlineCompletionStats?: { + lineCount: number + charCount: number + } + } +} + +export interface SuggestedState extends Omit { + phase: 'suggested' + /** Timestamp when the suggestion was first shown to the user. */ + suggestedAt: number + payload: PostProcessedState['payload'] +} + +export interface ReadState extends Omit { + phase: 'read' + /** Timestamp when the suggestion was marked as visible to the user. */ + readAt: number + payload: PostProcessedState['payload'] +} + +/** + * Common final payload properties shared between accepted and rejected states + */ +export type FinalPayload = PostProcessedState['payload'] & { + /** Displayed to the user for this many milliseconds. */ + timeFromSuggestedAt: number + /** True if the suggestion was explicitly/intentionally accepted. */ + isAccepted: boolean + /** + * True if the suggestion was visible for a certain time + * Required to correctly calculate CAR and other metrics where we + * want to account only for suggestions visible for a certain time. + * + * `timeFromSuggestedAt` is not a reliable source of truth for + * this case because a user could have rejected a suggestion without + * triggering `accepted` or `discarded` immediately. This is related to + * limited VS Code APIs which do not provide a reliable way to know + * if a suggestion is really visible. + */ + isRead: boolean + /** The number of the auto-edit started since the last suggestion was shown. */ + suggestionsStartedSinceLastSuggestion: number +} + +export interface AcceptedState extends Omit { + phase: 'accepted' + /** Timestamp when the user accepted the suggestion. */ + acceptedAt: number + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + /** Optional because it might be accepted before the read timeout */ + readAt?: number + payload: FinalPayload & Omit +} + +export interface RejectedState extends Omit { + phase: 'rejected' + /** Timestamp when the user rejected the suggestion. */ + rejectedAt: number + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + /** Optional because it might be accepted before the read timeout */ + readAt?: number + payload: FinalPayload +} + +export interface DiscardedState extends Omit { + phase: 'discarded' + /** Timestamp when the suggestion was discarded. */ + discardedAt: number + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + payload: StartedState['payload'] & { + discardReason: AutoeditDiscardReasonMetadata + } +} + +export interface PhaseStates { + started: StartedState + contextLoaded: ContextLoadedState + loaded: LoadedState + postProcessed: PostProcessedState + suggested: SuggestedState + read: ReadState + accepted: AcceptedState + rejected: RejectedState + discarded: DiscardedState +} + +export type AutoeditRequestState = PhaseStates[keyof PhaseStates] diff --git a/vscode/src/autoedits/autoedits-provider.ts b/vscode/src/autoedits/autoedits-provider.ts index fba6d924ebcc..037d857dd23e 100644 --- a/vscode/src/autoedits/autoedits-provider.ts +++ b/vscode/src/autoedits/autoedits-provider.ts @@ -296,10 +296,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v autoeditAnalyticsLogger.markAsLoaded({ requestId, prompt, + modelResponse: predictionResult, payload: { source: autoeditSource.network, isFuzzyMatch: false, - responseHeaders: predictionResult.responseHeaders, prediction: initialPrediction, }, }) From fa590872386dd62c571089cc94885be58801cb50 Mon Sep 17 00:00:00 2001 From: Valery Bugakov Date: Tue, 4 Mar 2025 12:17:00 +0800 Subject: [PATCH 3/4] feat(auto-edit): add basic webview debug-panel --- vscode/package.json | 18 ++ .../analytics-logger/analytics-logger.ts | 4 + vscode/src/autoedits/autoedits-config.ts | 2 +- .../autoedits/create-autoedits-provider.ts | 6 + .../src/autoedits/debug-panel/debug-panel.ts | 163 ++++++++++++++++++ .../autoedits/debug-panel/debug-protocol.ts | 15 ++ .../src/autoedits/debug-panel/debug-store.ts | 121 +++++++++++++ vscode/src/autoedits/debug-panel/index.ts | 2 + vscode/src/main.ts | 41 +++-- vscode/webviews/autoedit-debug.html | 17 ++ vscode/webviews/autoedit-debug.tsx | 94 ++++++++++ vscode/webviews/tailwind.config.mjs | 7 +- vscode/webviews/vite.config.mts | 1 + 13 files changed, 477 insertions(+), 14 deletions(-) create mode 100644 vscode/src/autoedits/debug-panel/debug-panel.ts create mode 100644 vscode/src/autoedits/debug-panel/debug-protocol.ts create mode 100644 vscode/src/autoedits/debug-panel/debug-store.ts create mode 100644 vscode/src/autoedits/debug-panel/index.ts create mode 100644 vscode/webviews/autoedit-debug.html create mode 100644 vscode/webviews/autoedit-debug.tsx diff --git a/vscode/package.json b/vscode/package.json index ea9b98a7d182..88a0f8d365d4 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -271,6 +271,14 @@ "group": "Cody", "icon": "$(feedback)" }, + { + "command": "cody.command.autoedit.open-debug-panel", + "title": "Debug Auto-Edit", + "category": "Cody", + "group": "Cody", + "icon": "$(debug)", + "enablement": "cody.activated" + }, { "command": "cody.command.explain-output", "title": "Ask Cody to Explain", @@ -582,6 +590,12 @@ "command": "cody.command.autoedit-manual-trigger", "title": "Autoedits Manual Trigger", "enablement": "cody.activated && config.cody.suggestions.mode == 'auto-edit (Experimental)'" + }, + { + "command": "cody.command.autoedit.open-debug-panel", + "category": "Cody", + "title": "Debug Auto-Edit", + "enablement": "cody.activated" } ], "keybindings": [ @@ -721,6 +735,10 @@ ], "menus": { "commandPalette": [ + { + "command": "cody.command.autoedit.open-debug-panel", + "when": "cody.activated" + }, { "command": "cody.command.edit-code", "when": "cody.activated && editorIsOpen" diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.ts index a3ee48a38a50..6367c14cfdac 100644 --- a/vscode/src/autoedits/analytics-logger/analytics-logger.ts +++ b/vscode/src/autoedits/analytics-logger/analytics-logger.ts @@ -25,6 +25,7 @@ import type { CodeToReplaceData } from '../prompt/prompt-utils' import type { DecorationInfo } from '../renderer/decorators/base' import { getDecorationStats } from '../renderer/diff-utils' +import { autoeditDebugStore } from '../debug-panel/debug-store' import { autoeditIdRegistry } from './suggestion-id-registry' import { type AcceptedState, @@ -358,6 +359,9 @@ export class AutoeditAnalyticsLogger { phase: nextPhase, } as PhaseStates[P] + // Integrate auto-edit analytics logger with the auto-edit debug panel. + autoeditDebugStore.addAutoeditRequestDebugState(updatedRequest) + this.activeRequests.set(requestId, updatedRequest) return { updatedRequest, currentRequest } diff --git a/vscode/src/autoedits/autoedits-config.ts b/vscode/src/autoedits/autoedits-config.ts index 39c9e9561b1e..6946cc4574e9 100644 --- a/vscode/src/autoedits/autoedits-config.ts +++ b/vscode/src/autoedits/autoedits-config.ts @@ -18,7 +18,7 @@ interface BaseAutoeditsProviderConfig { isChatModel: boolean } -interface AutoeditsProviderConfig extends BaseAutoeditsProviderConfig { +export interface AutoeditsProviderConfig extends BaseAutoeditsProviderConfig { experimentalAutoeditsConfigOverride: AutoEditsModelConfig | undefined isMockResponseFromCurrentDocumentTemplateEnabled: boolean } diff --git a/vscode/src/autoedits/create-autoedits-provider.ts b/vscode/src/autoedits/create-autoedits-provider.ts index b01d5f9762d3..d532631aa31c 100644 --- a/vscode/src/autoedits/create-autoedits-provider.ts +++ b/vscode/src/autoedits/create-autoedits-provider.ts @@ -20,6 +20,7 @@ import type { FixupController } from '../non-stop/FixupController' import type { CodyStatusBar } from '../services/StatusBar' import { AutoeditsProvider } from './autoedits-provider' +import { AutoeditDebugPanel } from './debug-panel/debug-panel' import { autoeditsOutputChannelLogger } from './output-channel-logger' import { initImageSuggestionService } from './renderer/image-gen' @@ -54,6 +55,7 @@ interface AutoeditsItemProviderArgs { autoeditImageRenderingEnabled: boolean fixupController: FixupController statusBar: CodyStatusBar + context: vscode.ExtensionContext } export function createAutoEditsProvider({ @@ -64,6 +66,7 @@ export function createAutoEditsProvider({ autoeditImageRenderingEnabled, fixupController, statusBar, + context, }: AutoeditsItemProviderArgs): Observable { if (!configuration.experimentalAutoEditEnabled) { return NEVER @@ -109,6 +112,9 @@ export function createAutoEditsProvider({ [{ scheme: 'file', language: '*' }, { notebookType: '*' }], provider ), + vscode.commands.registerCommand('cody.command.autoedit.open-debug-panel', () => { + AutoeditDebugPanel.showPanel(context) + }), provider, ] }), diff --git a/vscode/src/autoedits/debug-panel/debug-panel.ts b/vscode/src/autoedits/debug-panel/debug-panel.ts new file mode 100644 index 000000000000..771496794a07 --- /dev/null +++ b/vscode/src/autoedits/debug-panel/debug-panel.ts @@ -0,0 +1,163 @@ +import * as vscode from 'vscode' + +import { manipulateWebviewHTML } from '../../chat/chat-view/ChatController' +import type { AutoeditDebugMessageFromExtension } from './debug-protocol' + +import { autoeditDebugStore } from './debug-store' + +/** + * A panel that displays debug information about auto-edit requests. + */ +export class AutoeditDebugPanel { + public static currentPanel: AutoeditDebugPanel | undefined + private static readonly viewType = 'codyAutoeditDebugPabel' + + private readonly panel: vscode.WebviewPanel + private readonly extensionContext: vscode.ExtensionContext + private disposables: vscode.Disposable[] = [] + private updatePending = false + private readonly throttleMs = 500 // Throttle updates to at most once per 500ms + + private constructor(panel: vscode.WebviewPanel, extensionContext: vscode.ExtensionContext) { + this.panel = panel + this.extensionContext = extensionContext + + // Set the webview's initial content + void this.updateContent() + + // Listen for when the panel is disposed + // This happens when the user closes the panel or when the panel is closed programmatically + this.panel.onDidDispose(() => this.dispose(), null, this.disposables) + + // Subscribe to store changes with throttling + this.disposables.push( + autoeditDebugStore.onDidChange(() => { + // If an update is already pending, don't schedule another one + if (!this.updatePending) { + this.updatePending = true + setTimeout(() => { + this.updatePending = false + void this.updateContent() + }, this.throttleMs) + } + }) + ) + + // Handle messages from the webview + this.panel.webview.onDidReceiveMessage( + message => { + if (message.type === 'ready') { + // Send the initial data when the webview is ready + void this.updateContent() + } + }, + null, + this.disposables + ) + } + + /** + * Type-safe wrapper for sending messages to the webview. + * Ensures that only valid messages defined in the protocol are sent. + */ + private postMessageToWebview(message: AutoeditDebugMessageFromExtension): void { + this.panel.webview.postMessage(message) + } + + /** + * Shows the debug panel in the editor. + * If the panel already exists, it will be revealed. + */ + public static showPanel(extensionContext: vscode.ExtensionContext): void { + const column = vscode.window.activeTextEditor + ? vscode.window.activeTextEditor.viewColumn + : undefined + + // If we already have a panel, show it + if (AutoeditDebugPanel.currentPanel) { + AutoeditDebugPanel.currentPanel.panel.reveal(column) + return + } + + // Create a new panel + const panel = vscode.window.createWebviewPanel( + AutoeditDebugPanel.viewType, + 'Cody Auto-Edit Debug Panel', + column || vscode.ViewColumn.One, + { + // Enable JavaScript in the webview + enableScripts: true, + // Restrict the webview to only load resources from the extension's directory + localResourceRoots: [vscode.Uri.joinPath(extensionContext.extensionUri, 'dist')], + } + ) + + AutoeditDebugPanel.currentPanel = new AutoeditDebugPanel(panel, extensionContext) + } + + /** + * Updates the content of the panel with the latest auto-edit requests. + */ + private async updateContent(): Promise { + const entries = autoeditDebugStore.getAutoeditRequestDebugStates() + + // Send the updated entries to the webview using the type-safe protocol + this.postMessageToWebview({ + type: 'updateEntries', + entries, + }) + + // If no HTML content is set yet, set the initial HTML + if (!this.panel.webview.html) { + this.panel.webview.html = await this.getHtmlForWebview(this.panel.webview) + } + } + + /** + * Generates the HTML for the webview panel, including the React app. + */ + private async getHtmlForWebview(webview: vscode.Webview): Promise { + // Read the compiled HTML file using VS Code's file system API + try { + const htmlPath = vscode.Uri.joinPath( + this.extensionContext.extensionUri, + 'dist', + 'webviews', + 'autoedit-debug.html' + ) + const htmlBytes = await vscode.workspace.fs.readFile(htmlPath) + const htmlContent = new TextDecoder('utf-8').decode(htmlBytes) + + // Create URI for the webview resources + const webviewResourcesUri = webview.asWebviewUri( + vscode.Uri.joinPath(this.extensionContext.extensionUri, 'dist', 'webviews') + ) + + // Use the shared manipulateWebviewHTML function + return manipulateWebviewHTML(htmlContent, { + cspSource: webview.cspSource, + resources: webviewResourcesUri, + }) + } catch (error) { + console.error('Error getting HTML for webview:', error) + return '' + } + } + + /** + * Dispose of the panel when it's closed. + */ + public dispose(): void { + AutoeditDebugPanel.currentPanel = undefined + + // Clean up our resources + this.panel.dispose() + + while (this.disposables.length) { + const disposable = this.disposables.pop() + if (disposable) { + disposable.dispose() + } + } + } +} diff --git a/vscode/src/autoedits/debug-panel/debug-protocol.ts b/vscode/src/autoedits/debug-panel/debug-protocol.ts new file mode 100644 index 000000000000..e5deac432442 --- /dev/null +++ b/vscode/src/autoedits/debug-panel/debug-protocol.ts @@ -0,0 +1,15 @@ +import type { AutoeditRequestDebugState } from './debug-store' + +export type AutoeditDebugMessageFromExtension = { + type: 'updateEntries' + entries: ReadonlyArray +} + +export type AutoeditDebugMessageFromWebview = { type: 'ready' } + +export interface VSCodeAutoeditDebugWrapper { + postMessage: (message: AutoeditDebugMessageFromWebview) => void + onMessage: (callback: (message: AutoeditDebugMessageFromExtension) => void) => () => void + getState: () => unknown + setState: (newState: unknown) => unknown +} diff --git a/vscode/src/autoedits/debug-panel/debug-store.ts b/vscode/src/autoedits/debug-panel/debug-store.ts new file mode 100644 index 000000000000..c8d47a8f5e82 --- /dev/null +++ b/vscode/src/autoedits/debug-panel/debug-store.ts @@ -0,0 +1,121 @@ +import * as vscode from 'vscode' + +import type { AutoeditRequestState } from '../analytics-logger/types' +import { type AutoeditsProviderConfig, autoeditsProviderConfig } from '../autoedits-config' +import type { DecorationInfo } from '../renderer/decorators/base' +import { getDecorationInfo } from '../renderer/diff-utils' + +/** + * Enhanced debug entry for auto-edit requests that extends the analytics logger state + * with additional debug-specific properties. + */ +export interface AutoeditRequestDebugState { + /** The underlying analytics logger state object */ + state: AutoeditRequestState + /** Timestamp when the status was last updated */ + updatedAt: number + /** The autoedits provider config used for this request */ + autoeditsProviderConfig: AutoeditsProviderConfig + /** + * The side-by-side diff decoration info for the auto-edit request + * Different from the `state.updatedDecorationInfo` by the regex used to split + * the code in chunks for diffing it. + */ + sideBySideDiffDecorationInfo?: DecorationInfo +} + +const CHARACTER_REGEX = /./g + +/** + * A simple in-memory store for debugging auto-edit requests. + * Stores the most recent requests in a ring buffer. + */ +export class AutoeditDebugStore implements vscode.Disposable { + /** Auto-edit requests, stored in reverse chronological order (newest first) */ + private autoeditRequests: AutoeditRequestDebugState[] = [] + /** Maximum number of auto-edit requests to store */ + private maxEntries = 50 + /** Event emitter for notifying when data changes */ + private readonly onDidChangeEmitter = new vscode.EventEmitter() + /** Event that fires when the auto-edit requests data changes */ + public readonly onDidChange = this.onDidChangeEmitter.event + + /** + * Add a new auto-edit request debug state to the store. + * If the store is full, the oldest entry will be removed. + */ + public addAutoeditRequestDebugState(state: AutoeditRequestState): void { + const requestId = state.requestId + const existingIndex = this.autoeditRequests.findIndex( + entry => entry.state.requestId === requestId + ) + + if (existingIndex !== -1) { + this.updateExistingEntry(existingIndex, state) + } else { + this.addNewEntry(state) + } + } + + private updateExistingEntry(index: number, state: AutoeditRequestState): void { + const entry = this.autoeditRequests[index] + + this.autoeditRequests[index] = this.createDebugState(state, { + ...entry, + state, + sideBySideDiffDecorationInfo: this.calculateSideBySideDiff(state), + }) + + this.notifyChange() + } + + private addNewEntry(state: AutoeditRequestState): void { + const debugState = this.createDebugState(state) + this.autoeditRequests.unshift(debugState) + + this.enforceMaxEntries() + this.notifyChange() + } + + private createDebugState( + state: AutoeditRequestState, + baseState?: Partial + ): AutoeditRequestDebugState { + return { + state, + updatedAt: Date.now(), + autoeditsProviderConfig: { ...autoeditsProviderConfig }, + sideBySideDiffDecorationInfo: this.calculateSideBySideDiff(state), + ...baseState, + } + } + + private calculateSideBySideDiff(state: AutoeditRequestState): DecorationInfo | undefined { + // TODO: remove @ts-ignore once all auto-edit debug panel changes are merged. + return 'prediction' in state + ? // @ts-ignore + getDecorationInfo(state.codeToReplaceData.codeToRewrite, state.prediction, CHARACTER_REGEX) + : undefined + } + + private enforceMaxEntries(): void { + if (this.autoeditRequests.length > this.maxEntries) { + this.autoeditRequests = this.autoeditRequests.slice(0, this.maxEntries) + } + } + + private notifyChange(): void { + this.onDidChangeEmitter.fire() + } + + public getAutoeditRequestDebugStates(): ReadonlyArray { + return this.autoeditRequests + } + + public dispose(): void { + this.onDidChangeEmitter.dispose() + } +} + +// Singleton instance +export const autoeditDebugStore = new AutoeditDebugStore() diff --git a/vscode/src/autoedits/debug-panel/index.ts b/vscode/src/autoedits/debug-panel/index.ts new file mode 100644 index 000000000000..0becc4eab53a --- /dev/null +++ b/vscode/src/autoedits/debug-panel/index.ts @@ -0,0 +1,2 @@ +export { autoeditDebugStore, type AutoeditRequestDebugState } from './debug-store' +export { AutoeditDebugPanel } from './debug-panel' diff --git a/vscode/src/main.ts b/vscode/src/main.ts index a8e52d2c5541..3d7c2c181b36 100644 --- a/vscode/src/main.ts +++ b/vscode/src/main.ts @@ -53,6 +53,7 @@ import { tokenCallbackHandler, } from './auth/auth' import { createAutoEditsProvider } from './autoedits/create-autoedits-provider' +import { autoeditDebugStore } from './autoedits/debug-panel/debug-store' import { autoeditsOutputChannelLogger } from './autoedits/output-channel-logger' import { registerAutoEditTestRenderCommand } from './autoedits/renderer/mock-renderer' import type { MessageProviderOptions } from './chat/MessageProvider' @@ -292,7 +293,7 @@ const register = async ( registerAutocomplete(platform, statusBar, disposables) const tutorialSetup = tryRegisterTutorial(context, disposables) - await registerCodyCommands(statusBar, chatClient, fixupController, disposables) + await registerCodyCommands({ statusBar, chatClient, fixupController, disposables, context }) registerAuthCommands(disposables) registerChatCommands(disposables) disposables.push(...registerSidebarCommands()) @@ -413,12 +414,19 @@ async function registerOtherCommands(disposables: vscode.Disposable[]) { ) } -async function registerCodyCommands( - statusBar: CodyStatusBar, - chatClient: ChatClient, - fixupController: FixupController, +async function registerCodyCommands({ + statusBar, + chatClient, + fixupController, + disposables, + context, +}: { + statusBar: CodyStatusBar + chatClient: ChatClient + fixupController: FixupController disposables: vscode.Disposable[] -): Promise { + context: vscode.ExtensionContext +}): Promise { // Execute Cody Commands and Cody Custom Commands const executeCommand = ( commandKey: DefaultCodyCommands | string, @@ -462,7 +470,7 @@ async function registerCodyCommands( ) // Initialize autoedit provider if experimental feature is enabled - registerAutoEdits(chatClient, fixupController, statusBar, disposables) + registerAutoEdits({ chatClient, fixupController, statusBar, disposables, context }) // Initialize autoedit tester disposables.push( @@ -715,13 +723,21 @@ async function tryRegisterTutorial( } } -function registerAutoEdits( - chatClient: ChatClient, - fixupController: FixupController, - statusBar: CodyStatusBar, +function registerAutoEdits({ + chatClient, + fixupController, + statusBar, + disposables, + context, +}: { + chatClient: ChatClient + fixupController: FixupController + statusBar: CodyStatusBar disposables: vscode.Disposable[] -): void { + context: vscode.ExtensionContext +}): void { disposables.push( + autoeditDebugStore, subscriptionDisposable( combineLatest( resolvedConfig, @@ -754,6 +770,7 @@ function registerAutoEdits( autoeditImageRenderingEnabled, fixupController, statusBar, + context, }) } ), diff --git a/vscode/webviews/autoedit-debug.html b/vscode/webviews/autoedit-debug.html new file mode 100644 index 000000000000..e3c3d80b7633 --- /dev/null +++ b/vscode/webviews/autoedit-debug.html @@ -0,0 +1,17 @@ + + + + + + + + + + + Cody Auto-Edit Debug + + +
+ + + diff --git a/vscode/webviews/autoedit-debug.tsx b/vscode/webviews/autoedit-debug.tsx new file mode 100644 index 000000000000..911896f398c2 --- /dev/null +++ b/vscode/webviews/autoedit-debug.tsx @@ -0,0 +1,94 @@ +import { useEffect, useState } from 'react' +import { createRoot } from 'react-dom/client' + +import type { + AutoeditDebugMessageFromExtension, + VSCodeAutoeditDebugWrapper, +} from '../src/autoedits/debug-panel/debug-protocol' +import type { AutoeditRequestDebugState } from '../src/autoedits/debug-panel/debug-store' + +import { getVSCodeAPI } from './utils/VSCodeApi' + +/** + * Transforms array-formatted VS Code Range objects back into proper Range objects with start and end properties. + * + * @param obj The object to transform, which may contain Range objects that were converted to arrays. + * @returns A new object with any Range arrays converted back to objects with start and end properties. + */ +function transformRanges(obj: any): any { + if (obj === null || obj === undefined) { + return obj + } + + if (Array.isArray(obj)) { + // Check if it's a serialized Range in the format [{"line":1,"character":0},{"line":1,"character":4}] + if ( + obj.length === 2 && + typeof obj[0] === 'object' && + obj[0] !== null && + 'line' in obj[0] && + 'character' in obj[0] && + typeof obj[1] === 'object' && + obj[1] !== null && + 'line' in obj[1] && + 'character' in obj[1] + ) { + return { + start: { line: obj[0].line, character: obj[0].character }, + end: { line: obj[1].line, character: obj[1].character }, + } + } + // Otherwise, process each element of the array + return obj.map(item => transformRanges(item)) + } + + if (typeof obj === 'object') { + const result: Record = {} + for (const key in obj) { + result[key] = transformRanges(obj[key]) + } + return result + } + + return obj +} + +const vscode = getVSCodeAPI() as unknown as VSCodeAutoeditDebugWrapper + +function App() { + const [entries, setEntries] = useState([]) + + useEffect(() => { + // Listen for messages from VS Code + const handleMessage = (event: MessageEvent) => { + const message = event.data as AutoeditDebugMessageFromExtension + if (message.type === 'updateEntries') { + // Transform any Range arrays back to objects with start and end properties + const processedEntries = message.entries.map(entry => transformRanges(entry)) + + // Sort entries by updatedAt in descending order (newest first) + const sortedEntries = [...processedEntries].sort((a, b) => b.updatedAt - a.updatedAt) + setEntries(sortedEntries) + } + } + + window.addEventListener('message', handleMessage) + + // Request initial data + vscode.postMessage({ type: 'ready' }) + + return () => { + window.removeEventListener('message', handleMessage) + } + }, []) + + return ( +
+

Auto-Edit Debug Panel

+

We have {entries.length} entries

+
+ ) +} + +const root = createRoot(document.getElementById('root')!) +root.render() diff --git a/vscode/webviews/tailwind.config.mjs b/vscode/webviews/tailwind.config.mjs index bb0e4ee9d4ca..81224244df63 100644 --- a/vscode/webviews/tailwind.config.mjs +++ b/vscode/webviews/tailwind.config.mjs @@ -4,7 +4,12 @@ import plugin from 'tailwindcss/plugin' export default { content: { relative: true, - files: ['**/*.{ts,tsx}', '../../lib/**/**/*.{ts,tsx}'], + files: [ + '**/*.{ts,tsx}', + '../../lib/**/**/*.{ts,tsx}', + 'autoedit-debug/**/*.{ts,tsx}', + 'autoedit-debug/**/*.css', + ], }, prefix: 'tw-', theme: { diff --git a/vscode/webviews/vite.config.mts b/vscode/webviews/vite.config.mts index fa35a8187f18..f18785340300 100644 --- a/vscode/webviews/vite.config.mts +++ b/vscode/webviews/vite.config.mts @@ -26,6 +26,7 @@ export default defineProjectWithDefaults(__dirname, { }, input: { index: resolve(__dirname, 'index.html'), + autoeditDebug: resolve(__dirname, 'autoedit-debug.html'), }, output: { entryFileNames: '[name].js', From 0a14c0ec9135e81190248563d2572bf78799b9ac Mon Sep 17 00:00:00 2001 From: Valery Bugakov Date: Wed, 5 Mar 2025 10:52:27 +0800 Subject: [PATCH 4/4] feat(auto-edit): address review comments --- .../src/autoedits/debug-panel/debug-panel.ts | 31 ++++++++++++------- .../autoedits/debug-panel/debug-protocol.ts | 2 -- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/vscode/src/autoedits/debug-panel/debug-panel.ts b/vscode/src/autoedits/debug-panel/debug-panel.ts index 771496794a07..6243b5da0a04 100644 --- a/vscode/src/autoedits/debug-panel/debug-panel.ts +++ b/vscode/src/autoedits/debug-panel/debug-panel.ts @@ -10,7 +10,7 @@ import { autoeditDebugStore } from './debug-store' */ export class AutoeditDebugPanel { public static currentPanel: AutoeditDebugPanel | undefined - private static readonly viewType = 'codyAutoeditDebugPabel' + private static readonly viewType = 'codyAutoeditDebugPanel' private readonly panel: vscode.WebviewPanel private readonly extensionContext: vscode.ExtensionContext @@ -69,25 +69,34 @@ export class AutoeditDebugPanel { * If the panel already exists, it will be revealed. */ public static showPanel(extensionContext: vscode.ExtensionContext): void { - const column = vscode.window.activeTextEditor - ? vscode.window.activeTextEditor.viewColumn - : undefined - - // If we already have a panel, show it + // Try to reveal existing panel if available if (AutoeditDebugPanel.currentPanel) { - AutoeditDebugPanel.currentPanel.panel.reveal(column) - return + try { + const viewColumn = AutoeditDebugPanel.currentPanel.panel.viewColumn + AutoeditDebugPanel.currentPanel.panel.reveal(viewColumn, false) + return + } catch (error) { + console.log('Error revealing panel:', error) + AutoeditDebugPanel.currentPanel = undefined + } + } + + // Determine view column for a new panel + let viewColumn = vscode.ViewColumn.Beside + + if (vscode.window.activeTextEditor?.viewColumn === vscode.ViewColumn.One) { + viewColumn = vscode.ViewColumn.Two + } else if (vscode.window.activeTextEditor?.viewColumn) { + viewColumn = vscode.window.activeTextEditor.viewColumn } // Create a new panel const panel = vscode.window.createWebviewPanel( AutoeditDebugPanel.viewType, 'Cody Auto-Edit Debug Panel', - column || vscode.ViewColumn.One, + viewColumn, { - // Enable JavaScript in the webview enableScripts: true, - // Restrict the webview to only load resources from the extension's directory localResourceRoots: [vscode.Uri.joinPath(extensionContext.extensionUri, 'dist')], } ) diff --git a/vscode/src/autoedits/debug-panel/debug-protocol.ts b/vscode/src/autoedits/debug-panel/debug-protocol.ts index e5deac432442..69abe4b02b3b 100644 --- a/vscode/src/autoedits/debug-panel/debug-protocol.ts +++ b/vscode/src/autoedits/debug-panel/debug-protocol.ts @@ -10,6 +10,4 @@ export type AutoeditDebugMessageFromWebview = { type: 'ready' } export interface VSCodeAutoeditDebugWrapper { postMessage: (message: AutoeditDebugMessageFromWebview) => void onMessage: (callback: (message: AutoeditDebugMessageFromExtension) => void) => () => void - getState: () => unknown - setState: (newState: unknown) => unknown }