Dev Data Destinations

core/autocomplete/util/AutocompleteLoggingService.ts

@@ -1,4 +1,4 @@
-import { logDevData } from "../../util/devdata";
+import { DataLogger } from "../../data/log";
 import { COUNT_COMPLETION_REJECTED_AFTER } from "../../util/parameters";
 import { Telemetry } from "../../util/posthog";
 import { getUriFileExtension } from "../../util/uri";
@@ -96,7 +96,11 @@ export class AutocompleteLoggingService {
   }
 
   private logAutocompleteOutcome(outcome: AutocompleteOutcome) {
-    logDevData("autocomplete", outcome);
+    void DataLogger.getInstance().logDevData({
+      name: "autocomplete",
+      data: outcome,
+    });
+
     const { prompt, completion, prefix, suffix, ...restOfOutcome } = outcome;
     void Telemetry.capture(
       "autocomplete",

core/config/default.ts

@@ -97,11 +97,13 @@ export const defaultConfig: SerializedContinueConfig = {
   tabAutocompleteModel: DEFAULT_AUTOCOMPLETE_MODEL_CONFIG,
   contextProviders: defaultContextProvidersVsCode,
   slashCommands: defaultSlashCommandsVscode,
+  data: [],
 };
 
 export const defaultConfigJetBrains: SerializedContinueConfig = {
   models: [DEFAULT_CHAT_MODEL_CONFIG],
   tabAutocompleteModel: DEFAULT_AUTOCOMPLETE_MODEL_CONFIG,
   contextProviders: defaultContextProvidersJetBrains,
   slashCommands: defaultSlashCommandsJetBrains,
+  data: [],
 };

core/config/load.ts

@@ -584,6 +584,7 @@ async function finalToBrowserConfig(
     docs: final.docs,
     tools: final.tools,
     usePlatform: await useHub(ide.getIdeSettings()),
+    // data not included here
   };
 }
 

core/config/profile/PlatformProfileLoader.ts

@@ -1,9 +1,8 @@
-import { AssistantUnrolled } from "@continuedev/config-yaml/dist/schemas/index.js";
+import { AssistantUnrolled, ConfigResult } from "@continuedev/config-yaml";
 
 import { ControlPlaneClient } from "../../control-plane/client.js";
 import { ContinueConfig, IDE, IdeSettings } from "../../index.js";
 
-import { ConfigResult } from "@continuedev/config-yaml";
 import { ProfileDescription } from "../ProfileLifecycleManager.js";
 import doLoadConfig from "./doLoadConfig.js";
 import { IProfileLoader } from "./IProfileLoader.js";

core/config/yaml/convertFromJson.ts

@@ -1,5 +1,4 @@
-import { ModelConfig } from "@continuedev/config-yaml";
-import { AssistantUnrolled } from "@continuedev/config-yaml/dist/schemas";
+import { ModelConfig, AssistantUnrolled } from "@continuedev/config-yaml";
 
 import { SerializedContinueConfig } from "../..";
 

core/config/yaml/default.ts

@@ -1,4 +1,4 @@
-import { AssistantUnrolled } from "@continuedev/config-yaml/dist/schemas";
+import { AssistantUnrolled } from "@continuedev/config-yaml";
 
 // TODO
 export const defaultConfigYaml: AssistantUnrolled = {

core/config/yaml/loadYaml.ts

@@ -122,6 +122,7 @@ async function configYamlToContinueConfig(
       faviconUrl: doc.faviconUrl,
     })),
     contextProviders: [],
+    data: config.data,
   };
 
   // Models

core/core.ts

@@ -29,14 +29,14 @@ import { createNewPromptFileV2 } from "./promptFiles/v2/createNewPromptFile";
 import { callTool } from "./tools/callTool";
 import { ChatDescriber } from "./util/chatDescriber";
 import { clipboardCache } from "./util/clipboardCache";
-import { logDevData } from "./util/devdata";
-import { DevDataSqliteDb } from "./util/devdataSqlite";
+import { DataLogger } from "./data/log";
+import { DevDataSqliteDb } from "./data/devdataSqlite";
 import { GlobalContext } from "./util/GlobalContext";
 import historyManager from "./util/history";
 import {
   editConfigJson,
   getConfigJsonPath,
-  setupInitialDotContinueDirectory,
+  migrateV1DevDataFiles,
 } from "./util/paths";
 import { localPathToUri } from "./util/pathToUri";
 import { Telemetry } from "./util/posthog";
@@ -100,7 +100,7 @@ export class Core {
     private readonly onWrite: (text: string) => Promise<void> = async () => {},
   ) {
     // Ensure .continue directory is created
-    setupInitialDotContinueDirectory();
+    migrateV1DevDataFiles();
 
     this.codebaseIndexingState = {
       status: "loading",
@@ -144,6 +144,10 @@ export class Core {
       this.messenger.send("didChangeAvailableProfiles", { profiles }),
     );
 
+    // Dev Data Logger
+    const dataLogger = DataLogger.getInstance();
+    dataLogger.core = this;
+
     // Codebase Indexer and ContinueServerClient depend on IdeSettings
     let codebaseIndexerResolve: (_: any) => void | undefined;
     this.codebaseIndexerPromise = new Promise(
@@ -253,8 +257,8 @@ export class Core {
     });
 
     // Dev data
-    on("devdata/log", (msg) => {
-      logDevData(msg.data.tableName, msg.data.data);
+    on("devdata/log", async (msg) => {
+      void DataLogger.getInstance().logDevData(msg.data);
     });
 
     // Edit config

core/util/devdataSqlite.ts → core/data/devdataSqlite.ts

@@ -5,8 +5,9 @@ import sqlite3 from "sqlite3";
 
 import { DatabaseConnection } from "../indexing/refreshIndex.js";
 
-import { getDevDataSqlitePath } from "./paths.js";
+import { getDevDataSqlitePath } from "../util/paths.js";
 
+/* The Dev Data SQLITE table is only used for local tokens generated */
 export class DevDataSqliteDb {
   static db: DatabaseConnection | null = null;
 

core/data/log.ts

@@ -0,0 +1,144 @@
+import fs from "fs";
+import path from "path";
+
+import { getDevDataFilePath } from "../util/paths.js";
+import { Core } from "../core.js";
+import {
+  allDevEventNames,
+  DataLogLevel,
+  DevDataLogEvent,
+  devDataVersionedSchemas,
+} from "@continuedev/config-yaml";
+import { fetchwithRequestOptions } from "@continuedev/fetch";
+import { joinPathsToUri } from "../util/uri.js";
+import * as URI from "uri-js";
+import { fileURLToPath } from "url";
+
+const DEFAULT_DEV_DATA_LEVEL: DataLogLevel = "all";
+export const LOCAL_DEV_DATA_VERSION = "0.2.0";
+export class DataLogger {
+  private static instance: DataLogger | null = null;
+  core?: Core;
+
+  private constructor() {}
+
+  public static getInstance(): DataLogger {
+    if (DataLogger.instance === null) {
+      DataLogger.instance = new DataLogger();
+    }
+    return DataLogger.instance;
+  }
+
+  async logDevData(event: DevDataLogEvent) {
+    // Local logs (always on for all levels)
+    const filepath: string = getDevDataFilePath(
+      event.name,
+      LOCAL_DEV_DATA_VERSION,
+    );
+    const jsonLine = JSON.stringify(event.data);
+    fs.writeFileSync(filepath, `${jsonLine}\n`, { flag: "a" });
+
+    // Remote logs
+    const config = (await this.core?.configHandler.loadConfig())?.config;
+    if (config?.data?.length) {
+      await Promise.allSettled(
+        config.data.map(async (dataConfig) => {
+          try {
+            // First extract the data schema based on the version and level
+            const { schemaVersion } = dataConfig;
+
+            const level = dataConfig.level ?? DEFAULT_DEV_DATA_LEVEL;
+
+            // Skip event if `events` is specified and does not include the event
+            const events = dataConfig.events ?? allDevEventNames;
+            if (!events.includes(event.name)) {
+              return;
+            }
+
+            const versionSchemas = devDataVersionedSchemas[schemaVersion];
+            if (!versionSchemas) {
+              throw new Error(
+                `Attempting to log dev data to non-existent version ${schemaVersion}`,
+              );
+            }
+
+            const levelSchemas = versionSchemas[level];
+            if (!levelSchemas) {
+              throw new Error(
+                `Attempting to log dev data at level ${level} for version ${schemaVersion} which does not exist`,
+              );
+            }
+
+            const schema = levelSchemas[event.name];
+            if (!schema) {
+              throw new Error(
+                `Attempting to log dev data for event ${event.name} at level ${level} for version ${schemaVersion}: no schema found`,
+              );
+            }
+
+            const parsed = schema.safeParse(event.data);
+            if (!parsed.success) {
+              throw new Error(
+                `Failed to parse event data ${parsed.error.toString()}`,
+              );
+            }
+
+            const uriComponents = URI.parse(dataConfig.destination);
+
+            // Send to remote server
+            if (
+              uriComponents.scheme === "https" ||
+              uriComponents.scheme === "http"
+            ) {
+              const headers: Record<string, string> = {
+                "Content-Type": "application/json",
+              };
+              if (dataConfig.apiKey) {
+                headers["Authorization"] = `Bearer ${dataConfig.apiKey}`;
+              }
+              const response = await fetchwithRequestOptions(
+                dataConfig.destination,
+                {
+                  method: "POST",
+                  headers,
+                  body: JSON.stringify({
+                    name: event.name,
+                    data: parsed.data,
+                    schemaVersion,
+                    level,
+                  }),
+                },
+                dataConfig.requestOptions,
+              );
+              if (!response.ok) {
+                throw new Error(
+                  `Post request failed. ${response.status}: ${response.statusText}`,
+                );
+              }
+            } else if (uriComponents.scheme === "file") {
+              // Write to jsonc file for local file URIs
+              const dirUri = joinPathsToUri(
+                dataConfig.destination,
+                schemaVersion,
+              );
+              const dirPath = fileURLToPath(dirUri);
+
+              if (!fs.existsSync(dirPath)) {
+                fs.mkdirSync(dirPath, { recursive: true });
+              }
+              const filepath = path.join(dirPath, `${event.name}.jsonl`);
+              const jsonLine = JSON.stringify(event.data);
+              fs.writeFileSync(filepath, `${jsonLine}\n`, { flag: "a" });
+            } else {
+              throw new Error(`Unsupported scheme ${uriComponents.scheme}`);
+            }
+          } catch (error) {
+            console.error(
+              `Error logging data to ${dataConfig.destination}: ${error instanceof Error ? error.message : error}`,
+            );
+          }
+        }),
+      );
+    }
+  }
+}

core/index.d.ts

@@ -1,6 +1,6 @@
 import Parser from "web-tree-sitter";
 import { GetGhTokenArgs } from "./protocol/ide";
-
+import { DataDestination } from "@continuedev/config-yaml";
 declare global {
   interface Window {
     ide?: "vscode";
@@ -1134,6 +1134,7 @@ export interface SerializedContinueConfig {
   experimental?: ExperimentalConfig;
   analytics?: AnalyticsConfig;
   docs?: SiteIndexingConfig[];
+  data?: DataDestination[];
 }
 
 export type ConfigMergeType = "merge" | "overwrite";
@@ -1186,6 +1187,7 @@ export interface Config {
   experimental?: ExperimentalConfig;
   /** Analytics configuration */
   analytics?: AnalyticsConfig;
+  data?: DataDestination[];
 }
 
 // in the actual Continue source code
@@ -1209,6 +1211,7 @@ export interface ContinueConfig {
   analytics?: AnalyticsConfig;
   docs?: SiteIndexingConfig[];
   tools: Tool[];
+  data?: DataDestination[];
 }
 
 export interface BrowserSerializedContinueConfig {

core/llm/index.ts

@@ -21,8 +21,8 @@ import {
   RequestOptions,
   TemplateType,
 } from "../index.js";
-import { logDevData } from "../util/devdata.js";
-import { DevDataSqliteDb } from "../util/devdataSqlite.js";
+import { DataLogger } from "../data/log.js";
+import { DevDataSqliteDb } from "../data/devdataSqlite.js";
 import mergeJson from "../util/merge.js";
 import { renderChatMessage } from "../util/messageContent.js";
 import { isOllamaInstalled } from "../util/ollamaHelper.js";
@@ -339,11 +339,14 @@ export abstract class BaseLLM implements ILLM {
       generatedTokens,
     );
 
-    logDevData("tokens_generated", {
-      model: model,
-      provider: this.providerName,
-      promptTokens: promptTokens,
-      generatedTokens: generatedTokens,
+    void DataLogger.getInstance().logDevData({
+      name: "tokensGenerated",
+      data: {
+        model: model,
+        provider: this.providerName,
+        promptTokens: promptTokens,
+        generatedTokens: generatedTokens,
+      },
     });
   }