Merge pull request #3771 from continuedev/nate/yaml-loading

yaml updates

core/config/ConfigHandler.ts

@@ -14,9 +14,11 @@ import Ollama from "../llm/llms/Ollama.js";
 import { GlobalContext } from "../util/GlobalContext.js";
 import { getConfigJsonPath } from "../util/paths.js";
 
-import { ConfigResult } from "@continuedev/config-yaml";
+import { ConfigResult, ConfigYaml } from "@continuedev/config-yaml";
+import * as YAML from "yaml";
 import { controlPlaneEnv } from "../control-plane/env.js";
 import { usePlatform } from "../control-plane/flags.js";
+import { localPathToUri } from "../util/pathToUri.js";
 import {
   LOCAL_ONBOARDING_CHAT_MODEL,
   ONBOARDING_LOCAL_MODEL_TITLE,
@@ -28,7 +30,7 @@ import {
   ProfileDescription,
   ProfileLifecycleManager,
 } from "./ProfileLifecycleManager.js";
-import { localPathToUri } from "../util/pathToUri.js";
+import { clientRenderHelper } from "./yaml/clientRender.js";
 
 export type { ProfileDescription };
 
@@ -109,19 +111,35 @@ export class ConfigHandler {
         this.profiles = this.profiles.filter(
           (profile) => profile.profileDescription.id === "local",
         );
-        assistants.forEach((assistant) => {
-          const profileLoader = new PlatformProfileLoader(
-            assistant.configResult,
-            assistant.ownerSlug,
-            assistant.packageSlug,
-            this.controlPlaneClient,
-            this.ide,
-            this.ideSettingsPromise,
-            this.writeLog,
-            this.reloadConfig.bind(this),
-          );
-          this.profiles.push(new ProfileLifecycleManager(profileLoader));
-        });
+        await Promise.all(
+          assistants.map(async (assistant) => {
+            let renderedConfig: ConfigYaml | undefined = undefined;
+            if (assistant.configResult.config) {
+              renderedConfig = await clientRenderHelper(
+                YAML.stringify(assistant.configResult.config),
+                this.ide,
+                this.controlPlaneClient,
+              );
+            }
+
+            const profileLoader = new PlatformProfileLoader(
+              { ...assistant.configResult, config: renderedConfig },
+              assistant.ownerSlug,
+              assistant.packageSlug,
+              this.controlPlaneClient,
+              this.ide,
+              this.ideSettingsPromise,
+              this.writeLog,
+              this.reloadConfig.bind(this),
+            );
+            this.profiles = [
+              ...this.profiles.filter(
+                (profile) => profile.profileDescription.id === "local",
+              ),
+              new ProfileLifecycleManager(profileLoader),
+            ];
+          }),
+        );
 
         this.notifyProfileListeners(
           this.profiles.map((profile) => profile.profileDescription),

core/config/profile/PlatformProfileLoader.ts

@@ -1,10 +1,12 @@
-import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js";
+import { ConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js";
+import * as YAML from "yaml";
 
 import { ControlPlaneClient } from "../../control-plane/client.js";
 import { ContinueConfig, IDE, IdeSettings } from "../../index.js";
 
 import { ConfigResult } from "@continuedev/config-yaml";
 import { ProfileDescription } from "../ProfileLifecycleManager.js";
+import { clientRenderHelper } from "../yaml/clientRender.js";
 import doLoadConfig from "./doLoadConfig.js";
 import { IProfileLoader } from "./IProfileLoader.js";
 
@@ -24,7 +26,7 @@ export default class PlatformProfileLoader implements IProfileLoader {
   description: ProfileDescription;
 
   constructor(
-    private configResult: ConfigResult<ClientConfigYaml>,
+    private configResult: ConfigResult<ConfigYaml>,
     private readonly ownerSlug: string,
     private readonly packageSlug: string,
     private readonly controlPlaneClient: ControlPlaneClient,
@@ -49,8 +51,18 @@ export default class PlatformProfileLoader implements IProfileLoader {
       if (!newConfigResult) {
         return;
       }
+
+      let renderedConfig: ConfigYaml | undefined = undefined;
+      if (newConfigResult.config) {
+        renderedConfig = await clientRenderHelper(
+          YAML.stringify(newConfigResult.config),
+          this.ide,
+          this.controlPlaneClient,
+        );
+      }
+
       this.configResult = {
-        config: newConfigResult.config,
+        config: renderedConfig,
         errors: newConfigResult.errors,
         configLoadInterrupted: false,
       };

core/config/profile/doLoadConfig.ts

@@ -1,7 +1,10 @@
 import fs from "fs";
 
-import { ConfigResult, ConfigValidationError } from "@continuedev/config-yaml";
-import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas";
+import {
+  ConfigResult,
+  ConfigValidationError,
+  ConfigYaml,
+} from "@continuedev/config-yaml";
 import {
   ContinueConfig,
   ContinueRcJson,
@@ -27,7 +30,7 @@ export default async function doLoadConfig(
   controlPlaneClient: ControlPlaneClient,
   writeLog: (message: string) => Promise<void>,
   overrideConfigJson: SerializedContinueConfig | undefined,
-  overrideConfigYaml: ClientConfigYaml | undefined,
+  overrideConfigYaml: ConfigYaml | undefined,
   platformConfigMetadata: PlatformConfigMetadata | undefined,
   workspaceId?: string,
 ): Promise<ConfigResult<ContinueConfig>> {

core/config/yaml/clientRender.ts

@@ -0,0 +1,40 @@
+import {
+  clientRender,
+  PlatformClient,
+  SecretStore,
+} from "@continuedev/config-yaml";
+
+import { IDE } from "../..";
+import { ControlPlaneClient } from "../../control-plane/client";
+
+export async function clientRenderHelper(
+  unrolledAssistant: string,
+  ide: IDE,
+  controlPlaneClient: ControlPlaneClient,
+) {
+  const ideSecretStore: SecretStore = {
+    get: async function (secretName: string): Promise<string | undefined> {
+      const results = await ide.readSecrets([secretName]);
+      return results[secretName];
+    },
+    set: async function (
+      secretName: string,
+      secretValue: string,
+    ): Promise<void> {
+      return await ide.writeSecrets({
+        [secretName]: secretValue,
+      });
+    },
+  };
+
+  const platformClient: PlatformClient = {
+    resolveFQSNs: controlPlaneClient.resolveFQSNs.bind(controlPlaneClient),
+  };
+
+  const userId = await controlPlaneClient.userId;
+  return await clientRender(
+    unrolledAssistant,
+    ideSecretStore,
+    userId ? platformClient : undefined,
+  );
+}

core/config/yaml/loadYaml.ts

@@ -2,13 +2,10 @@ import fs from "node:fs";
 
 import {
   ConfigResult,
-  fillTemplateVariables,
-  resolveSecretsOnClient,
+  ConfigYaml,
   validateConfigYaml,
 } from "@continuedev/config-yaml";
-import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas";
 import { fetchwithRequestOptions } from "@continuedev/fetch";
-import * as YAML from "yaml";
 
 import {
   BrowserSerializedContinueConfig,
@@ -21,44 +18,34 @@ import {
 } from "../..";
 import { AllRerankers } from "../../context/allRerankers";
 import { MCPManagerSingleton } from "../../context/mcp";
+import CodebaseContextProvider from "../../context/providers/CodebaseContextProvider";
+import FileContextProvider from "../../context/providers/FileContextProvider";
 import { contextProviderClassFromName } from "../../context/providers/index";
+import PromptFilesContextProvider from "../../context/providers/PromptFilesContextProvider";
+import { ControlPlaneClient } from "../../control-plane/client";
 import { allEmbeddingsProviders } from "../../indexing/allEmbeddingsProviders";
 import FreeTrial from "../../llm/llms/FreeTrial";
 import TransformersJsEmbeddingsProvider from "../../llm/llms/TransformersJsEmbeddingsProvider";
 import { slashCommandFromPromptFileV1 } from "../../promptFiles/v1/slashCommandFromPromptFile";
 import { getAllPromptFiles } from "../../promptFiles/v2/getPromptFiles";
-import { getConfigYamlPath, getContinueDotEnv } from "../../util/paths";
+import { getConfigYamlPath } from "../../util/paths";
 import { getSystemPromptDotFile } from "../getSystemPromptDotFile";
 import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader";
 
-import CodebaseContextProvider from "../../context/providers/CodebaseContextProvider";
-import FileContextProvider from "../../context/providers/FileContextProvider";
-import PromptFilesContextProvider from "../../context/providers/PromptFilesContextProvider";
-import { ControlPlaneClient } from "../../control-plane/client";
+import { allTools } from "../../tools";
+import { clientRenderHelper } from "./clientRender";
 import { llmsFromModelConfig } from "./models";
 
-function renderTemplateVars(configYaml: string): string {
-  const data: Record<string, string> = {};
-
-  // env.*
-  const envVars = getContinueDotEnv();
-  Object.entries(envVars).forEach(([key, value]) => {
-    data[`env.${key}`] = value;
-  });
-
-  // secrets.* not filled in
-
-  return fillTemplateVariables(configYaml, data);
-}
-
-function loadConfigYaml(
+async function loadConfigYaml(
   workspaceConfigs: string[],
   rawYaml: string,
-  overrideConfigYaml: ClientConfigYaml | undefined,
-): ConfigResult<ClientConfigYaml> {
+  overrideConfigYaml: ConfigYaml | undefined,
+  ide: IDE,
+  controlPlaneClient: ControlPlaneClient,
+): Promise<ConfigResult<ConfigYaml>> {
   let config =
     overrideConfigYaml ??
-    (YAML.parse(renderTemplateVars(rawYaml)) as ClientConfigYaml);
+    (await clientRenderHelper(rawYaml, ide, controlPlaneClient));
   const errors = validateConfigYaml(config);
 
   if (errors?.some((error) => error.fatal)) {
@@ -98,7 +85,7 @@ async function slashCommandsFromV1PromptFiles(
 }
 
 async function configYamlToContinueConfig(
-  config: ClientConfigYaml,
+  config: ConfigYaml,
   ide: IDE,
   ideSettings: IdeSettings,
   uniqueId: string,
@@ -111,7 +98,7 @@ async function configYamlToContinueConfig(
     slashCommands: await slashCommandsFromV1PromptFiles(ide),
     models: [],
     tabAutocompleteModels: [],
-    tools: [],
+    tools: allTools,
     systemMessage: config.rules?.join("\n"),
     embeddingsProvider: new TransformersJsEmbeddingsProvider(),
     experimental: {
@@ -304,7 +291,7 @@ export async function loadContinueConfigFromYaml(
   uniqueId: string,
   writeLog: (log: string) => Promise<void>,
   workOsAccessToken: string | undefined,
-  overrideConfigYaml: ClientConfigYaml | undefined,
+  overrideConfigYaml: ConfigYaml | undefined,
   platformConfigMetadata: PlatformConfigMetadata | undefined,
   controlPlaneClient: ControlPlaneClient,
 ): Promise<ConfigResult<ContinueConfig>> {
@@ -314,10 +301,12 @@ export async function loadContinueConfigFromYaml(
       ? fs.readFileSync(configYamlPath, "utf-8")
       : "";
 
-  const configYamlResult = loadConfigYaml(
+  const configYamlResult = await loadConfigYaml(
     workspaceConfigs,
     rawYaml,
     overrideConfigYaml,
+    ide,
+    controlPlaneClient,
   );
 
   if (!configYamlResult.config || configYamlResult.configLoadInterrupted) {
@@ -328,18 +317,8 @@ export async function loadContinueConfigFromYaml(
     };
   }
 
-  const configYaml = await resolveSecretsOnClient(
-    configYamlResult.config,
-    ide.readSecrets.bind(ide),
-    async (secretNames: string[]) => {
-      const secretValues = await controlPlaneClient.syncSecrets(secretNames);
-      await ide.writeSecrets(secretValues);
-      return secretValues;
-    },
-  );
-
   const continueConfig = await configYamlToContinueConfig(
-    configYaml,
+    configYamlResult.config,
     ide,
     ideSettings,
     uniqueId,

core/config/yaml/models.ts

@@ -3,25 +3,13 @@ import { ModelConfig } from "@continuedev/config-yaml";
 import { IDE, IdeSettings, LLMOptions } from "../..";
 import { BaseLLM } from "../../llm";
 import { LLMClasses } from "../../llm/llms";
-import ContinueProxy from "../../llm/llms/stubs/ContinueProxy";
 import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader";
 
 const AUTODETECT = "AUTODETECT";
 
-function useContinueProxy(
-  model: ModelConfig,
-  platformConfigMetadata: PlatformConfigMetadata | undefined,
-): boolean {
-  return !!platformConfigMetadata && model.apiKeySecret !== undefined;
-}
-
 function getModelClass(
   model: ModelConfig,
-  platformConfigMetadata: PlatformConfigMetadata | undefined,
 ): (typeof LLMClasses)[number] | undefined {
-  if (useContinueProxy(model, platformConfigMetadata)) {
-    return ContinueProxy;
-  }
   return LLMClasses.find((llm) => llm.providerName === model.provider);
 }
 
@@ -41,13 +29,13 @@ async function modelConfigToBaseLLM(
   platformConfigMetadata: PlatformConfigMetadata | undefined,
   systemMessage: string | undefined,
 ): Promise<BaseLLM | undefined> {
-  const cls = getModelClass(model, platformConfigMetadata);
+  const cls = getModelClass(model);
 
   if (!cls) {
     return undefined;
   }
 
-  const usingContinueProxy = useContinueProxy(model, platformConfigMetadata);
+  const usingContinueProxy = model.provider === "continue-proxy";
   const modelName = usingContinueProxy
     ? getContinueProxyModelName(
         platformConfigMetadata!.ownerSlug,

core/control-plane/client.ts

@@ -1,10 +1,10 @@
 import { ConfigJson } from "@continuedev/config-types";
-import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js";
+import { ConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js";
 import fetch, { RequestInit, Response } from "node-fetch";
 
 import { ModelDescription } from "../index.js";
 
-import { ConfigResult } from "@continuedev/config-yaml";
+import { ConfigResult, FQSN, SecretResult } from "@continuedev/config-yaml";
 import { controlPlaneEnv } from "./env.js";
 
 export interface ControlPlaneSessionInfo {
@@ -38,6 +38,19 @@ export class ControlPlaneClient {
     >,
   ) {}
 
+  async resolveFQSNs(fqsns: FQSN[]): Promise<(SecretResult | undefined)[]> {
+    const userId = await this.userId;
+    if (!userId) {
+      throw new Error("No user id");
+    }
+
+    const resp = await this.request("ide/sync-secrets", {
+      method: "POST",
+      body: JSON.stringify({ fqsns }),
+    });
+    return (await resp.json()) as any;
+  }
+
   get userId(): Promise<string | undefined> {
     return this.sessionInfoPromise.then(
       (sessionInfo) => sessionInfo?.account.id,
@@ -89,7 +102,7 @@ export class ControlPlaneClient {
 
   public async listAssistants(): Promise<
     {
-      configResult: ConfigResult<ClientConfigYaml>;
+      configResult: ConfigResult<ConfigYaml>;
       ownerSlug: string;
       packageSlug: string;
       iconUrl: string;