diff --git a/core/config/ConfigHandler.ts b/core/config/ConfigHandler.ts index 626e237dbf..24987528a3 100644 --- a/core/config/ConfigHandler.ts +++ b/core/config/ConfigHandler.ts @@ -14,9 +14,11 @@ import Ollama from "../llm/llms/Ollama.js"; import { GlobalContext } from "../util/GlobalContext.js"; import { getConfigJsonPath } from "../util/paths.js"; -import { ConfigResult } from "@continuedev/config-yaml"; +import { ConfigResult, ConfigYaml } from "@continuedev/config-yaml"; +import * as YAML from "yaml"; import { controlPlaneEnv } from "../control-plane/env.js"; import { usePlatform } from "../control-plane/flags.js"; +import { localPathToUri } from "../util/pathToUri.js"; import { LOCAL_ONBOARDING_CHAT_MODEL, ONBOARDING_LOCAL_MODEL_TITLE, @@ -28,7 +30,7 @@ import { ProfileDescription, ProfileLifecycleManager, } from "./ProfileLifecycleManager.js"; -import { localPathToUri } from "../util/pathToUri.js"; +import { clientRenderHelper } from "./yaml/clientRender.js"; export type { ProfileDescription }; @@ -109,19 +111,35 @@ export class ConfigHandler { this.profiles = this.profiles.filter( (profile) => profile.profileDescription.id === "local", ); - assistants.forEach((assistant) => { - const profileLoader = new PlatformProfileLoader( - assistant.configResult, - assistant.ownerSlug, - assistant.packageSlug, - this.controlPlaneClient, - this.ide, - this.ideSettingsPromise, - this.writeLog, - this.reloadConfig.bind(this), - ); - this.profiles.push(new ProfileLifecycleManager(profileLoader)); - }); + await Promise.all( + assistants.map(async (assistant) => { + let renderedConfig: ConfigYaml | undefined = undefined; + if (assistant.configResult.config) { + renderedConfig = await clientRenderHelper( + YAML.stringify(assistant.configResult.config), + this.ide, + this.controlPlaneClient, + ); + } + + const profileLoader = new PlatformProfileLoader( + { ...assistant.configResult, config: renderedConfig }, + assistant.ownerSlug, + assistant.packageSlug, + this.controlPlaneClient, + this.ide, + this.ideSettingsPromise, + this.writeLog, + this.reloadConfig.bind(this), + ); + this.profiles = [ + ...this.profiles.filter( + (profile) => profile.profileDescription.id === "local", + ), + new ProfileLifecycleManager(profileLoader), + ]; + }), + ); this.notifyProfileListeners( this.profiles.map((profile) => profile.profileDescription), diff --git a/core/config/profile/PlatformProfileLoader.ts b/core/config/profile/PlatformProfileLoader.ts index d0f7d6f802..5319097b36 100644 --- a/core/config/profile/PlatformProfileLoader.ts +++ b/core/config/profile/PlatformProfileLoader.ts @@ -1,10 +1,12 @@ -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; +import { ConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; +import * as YAML from "yaml"; import { ControlPlaneClient } from "../../control-plane/client.js"; import { ContinueConfig, IDE, IdeSettings } from "../../index.js"; import { ConfigResult } from "@continuedev/config-yaml"; import { ProfileDescription } from "../ProfileLifecycleManager.js"; +import { clientRenderHelper } from "../yaml/clientRender.js"; import doLoadConfig from "./doLoadConfig.js"; import { IProfileLoader } from "./IProfileLoader.js"; @@ -24,7 +26,7 @@ export default class PlatformProfileLoader implements IProfileLoader { description: ProfileDescription; constructor( - private configResult: ConfigResult, + private configResult: ConfigResult, private readonly ownerSlug: string, private readonly packageSlug: string, private readonly controlPlaneClient: ControlPlaneClient, @@ -49,8 +51,18 @@ export default class PlatformProfileLoader implements IProfileLoader { if (!newConfigResult) { return; } + + let renderedConfig: ConfigYaml | undefined = undefined; + if (newConfigResult.config) { + renderedConfig = await clientRenderHelper( + YAML.stringify(newConfigResult.config), + this.ide, + this.controlPlaneClient, + ); + } + this.configResult = { - config: newConfigResult.config, + config: renderedConfig, errors: newConfigResult.errors, configLoadInterrupted: false, }; diff --git a/core/config/profile/doLoadConfig.ts b/core/config/profile/doLoadConfig.ts index 5f20ac7e29..8e2326e3db 100644 --- a/core/config/profile/doLoadConfig.ts +++ b/core/config/profile/doLoadConfig.ts @@ -1,7 +1,10 @@ import fs from "fs"; -import { ConfigResult, ConfigValidationError } from "@continuedev/config-yaml"; -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas"; +import { + ConfigResult, + ConfigValidationError, + ConfigYaml, +} from "@continuedev/config-yaml"; import { ContinueConfig, ContinueRcJson, @@ -27,7 +30,7 @@ export default async function doLoadConfig( controlPlaneClient: ControlPlaneClient, writeLog: (message: string) => Promise, overrideConfigJson: SerializedContinueConfig | undefined, - overrideConfigYaml: ClientConfigYaml | undefined, + overrideConfigYaml: ConfigYaml | undefined, platformConfigMetadata: PlatformConfigMetadata | undefined, workspaceId?: string, ): Promise> { diff --git a/core/config/yaml/clientRender.ts b/core/config/yaml/clientRender.ts new file mode 100644 index 0000000000..8cb727456f --- /dev/null +++ b/core/config/yaml/clientRender.ts @@ -0,0 +1,40 @@ +import { + clientRender, + PlatformClient, + SecretStore, +} from "@continuedev/config-yaml"; + +import { IDE } from "../.."; +import { ControlPlaneClient } from "../../control-plane/client"; + +export async function clientRenderHelper( + unrolledAssistant: string, + ide: IDE, + controlPlaneClient: ControlPlaneClient, +) { + const ideSecretStore: SecretStore = { + get: async function (secretName: string): Promise { + const results = await ide.readSecrets([secretName]); + return results[secretName]; + }, + set: async function ( + secretName: string, + secretValue: string, + ): Promise { + return await ide.writeSecrets({ + [secretName]: secretValue, + }); + }, + }; + + const platformClient: PlatformClient = { + resolveFQSNs: controlPlaneClient.resolveFQSNs.bind(controlPlaneClient), + }; + + const userId = await controlPlaneClient.userId; + return await clientRender( + unrolledAssistant, + ideSecretStore, + userId ? platformClient : undefined, + ); +} diff --git a/core/config/yaml/loadYaml.ts b/core/config/yaml/loadYaml.ts index 5e0161ad80..09003f6eb4 100644 --- a/core/config/yaml/loadYaml.ts +++ b/core/config/yaml/loadYaml.ts @@ -2,13 +2,10 @@ import fs from "node:fs"; import { ConfigResult, - fillTemplateVariables, - resolveSecretsOnClient, + ConfigYaml, validateConfigYaml, } from "@continuedev/config-yaml"; -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas"; import { fetchwithRequestOptions } from "@continuedev/fetch"; -import * as YAML from "yaml"; import { BrowserSerializedContinueConfig, @@ -21,44 +18,34 @@ import { } from "../.."; import { AllRerankers } from "../../context/allRerankers"; import { MCPManagerSingleton } from "../../context/mcp"; +import CodebaseContextProvider from "../../context/providers/CodebaseContextProvider"; +import FileContextProvider from "../../context/providers/FileContextProvider"; import { contextProviderClassFromName } from "../../context/providers/index"; +import PromptFilesContextProvider from "../../context/providers/PromptFilesContextProvider"; +import { ControlPlaneClient } from "../../control-plane/client"; import { allEmbeddingsProviders } from "../../indexing/allEmbeddingsProviders"; import FreeTrial from "../../llm/llms/FreeTrial"; import TransformersJsEmbeddingsProvider from "../../llm/llms/TransformersJsEmbeddingsProvider"; import { slashCommandFromPromptFileV1 } from "../../promptFiles/v1/slashCommandFromPromptFile"; import { getAllPromptFiles } from "../../promptFiles/v2/getPromptFiles"; -import { getConfigYamlPath, getContinueDotEnv } from "../../util/paths"; +import { getConfigYamlPath } from "../../util/paths"; import { getSystemPromptDotFile } from "../getSystemPromptDotFile"; import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader"; -import CodebaseContextProvider from "../../context/providers/CodebaseContextProvider"; -import FileContextProvider from "../../context/providers/FileContextProvider"; -import PromptFilesContextProvider from "../../context/providers/PromptFilesContextProvider"; -import { ControlPlaneClient } from "../../control-plane/client"; +import { allTools } from "../../tools"; +import { clientRenderHelper } from "./clientRender"; import { llmsFromModelConfig } from "./models"; -function renderTemplateVars(configYaml: string): string { - const data: Record = {}; - - // env.* - const envVars = getContinueDotEnv(); - Object.entries(envVars).forEach(([key, value]) => { - data[`env.${key}`] = value; - }); - - // secrets.* not filled in - - return fillTemplateVariables(configYaml, data); -} - -function loadConfigYaml( +async function loadConfigYaml( workspaceConfigs: string[], rawYaml: string, - overrideConfigYaml: ClientConfigYaml | undefined, -): ConfigResult { + overrideConfigYaml: ConfigYaml | undefined, + ide: IDE, + controlPlaneClient: ControlPlaneClient, +): Promise> { let config = overrideConfigYaml ?? - (YAML.parse(renderTemplateVars(rawYaml)) as ClientConfigYaml); + (await clientRenderHelper(rawYaml, ide, controlPlaneClient)); const errors = validateConfigYaml(config); if (errors?.some((error) => error.fatal)) { @@ -98,7 +85,7 @@ async function slashCommandsFromV1PromptFiles( } async function configYamlToContinueConfig( - config: ClientConfigYaml, + config: ConfigYaml, ide: IDE, ideSettings: IdeSettings, uniqueId: string, @@ -111,7 +98,7 @@ async function configYamlToContinueConfig( slashCommands: await slashCommandsFromV1PromptFiles(ide), models: [], tabAutocompleteModels: [], - tools: [], + tools: allTools, systemMessage: config.rules?.join("\n"), embeddingsProvider: new TransformersJsEmbeddingsProvider(), experimental: { @@ -304,7 +291,7 @@ export async function loadContinueConfigFromYaml( uniqueId: string, writeLog: (log: string) => Promise, workOsAccessToken: string | undefined, - overrideConfigYaml: ClientConfigYaml | undefined, + overrideConfigYaml: ConfigYaml | undefined, platformConfigMetadata: PlatformConfigMetadata | undefined, controlPlaneClient: ControlPlaneClient, ): Promise> { @@ -314,10 +301,12 @@ export async function loadContinueConfigFromYaml( ? fs.readFileSync(configYamlPath, "utf-8") : ""; - const configYamlResult = loadConfigYaml( + const configYamlResult = await loadConfigYaml( workspaceConfigs, rawYaml, overrideConfigYaml, + ide, + controlPlaneClient, ); if (!configYamlResult.config || configYamlResult.configLoadInterrupted) { @@ -328,18 +317,8 @@ export async function loadContinueConfigFromYaml( }; } - const configYaml = await resolveSecretsOnClient( - configYamlResult.config, - ide.readSecrets.bind(ide), - async (secretNames: string[]) => { - const secretValues = await controlPlaneClient.syncSecrets(secretNames); - await ide.writeSecrets(secretValues); - return secretValues; - }, - ); - const continueConfig = await configYamlToContinueConfig( - configYaml, + configYamlResult.config, ide, ideSettings, uniqueId, diff --git a/core/config/yaml/models.ts b/core/config/yaml/models.ts index 309e061d19..8fe34677f4 100644 --- a/core/config/yaml/models.ts +++ b/core/config/yaml/models.ts @@ -3,25 +3,13 @@ import { ModelConfig } from "@continuedev/config-yaml"; import { IDE, IdeSettings, LLMOptions } from "../.."; import { BaseLLM } from "../../llm"; import { LLMClasses } from "../../llm/llms"; -import ContinueProxy from "../../llm/llms/stubs/ContinueProxy"; import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader"; const AUTODETECT = "AUTODETECT"; -function useContinueProxy( - model: ModelConfig, - platformConfigMetadata: PlatformConfigMetadata | undefined, -): boolean { - return !!platformConfigMetadata && model.apiKeySecret !== undefined; -} - function getModelClass( model: ModelConfig, - platformConfigMetadata: PlatformConfigMetadata | undefined, ): (typeof LLMClasses)[number] | undefined { - if (useContinueProxy(model, platformConfigMetadata)) { - return ContinueProxy; - } return LLMClasses.find((llm) => llm.providerName === model.provider); } @@ -41,13 +29,13 @@ async function modelConfigToBaseLLM( platformConfigMetadata: PlatformConfigMetadata | undefined, systemMessage: string | undefined, ): Promise { - const cls = getModelClass(model, platformConfigMetadata); + const cls = getModelClass(model); if (!cls) { return undefined; } - const usingContinueProxy = useContinueProxy(model, platformConfigMetadata); + const usingContinueProxy = model.provider === "continue-proxy"; const modelName = usingContinueProxy ? getContinueProxyModelName( platformConfigMetadata!.ownerSlug, diff --git a/core/control-plane/client.ts b/core/control-plane/client.ts index 3a29b3fee9..bb7661ee83 100644 --- a/core/control-plane/client.ts +++ b/core/control-plane/client.ts @@ -1,10 +1,10 @@ import { ConfigJson } from "@continuedev/config-types"; -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; +import { ConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; import fetch, { RequestInit, Response } from "node-fetch"; import { ModelDescription } from "../index.js"; -import { ConfigResult } from "@continuedev/config-yaml"; +import { ConfigResult, FQSN, SecretResult } from "@continuedev/config-yaml"; import { controlPlaneEnv } from "./env.js"; export interface ControlPlaneSessionInfo { @@ -38,6 +38,19 @@ export class ControlPlaneClient { >, ) {} + async resolveFQSNs(fqsns: FQSN[]): Promise<(SecretResult | undefined)[]> { + const userId = await this.userId; + if (!userId) { + throw new Error("No user id"); + } + + const resp = await this.request("ide/sync-secrets", { + method: "POST", + body: JSON.stringify({ fqsns }), + }); + return (await resp.json()) as any; + } + get userId(): Promise { return this.sessionInfoPromise.then( (sessionInfo) => sessionInfo?.account.id, @@ -89,7 +102,7 @@ export class ControlPlaneClient { public async listAssistants(): Promise< { - configResult: ConfigResult; + configResult: ConfigResult; ownerSlug: string; packageSlug: string; iconUrl: string; diff --git a/core/index.d.ts b/core/index.d.ts index 098628c0fd..63847aa14b 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -460,7 +460,7 @@ export interface LLMOptions { writeLog?: (str: string) => Promise; llmRequestHook?: (model: string, prompt: string) => any; apiKey?: string; - apiKeySecret?: string; + apiKeyLocation?: string; aiGatewaySlug?: string; apiBase?: string; cacheBehavior?: CacheBehavior; @@ -896,7 +896,7 @@ export interface ModelDescription { provider: string; model: string; apiKey?: string; - apiKeySecret?: string; + apiKeyLocation?: string; apiBase?: string; contextLength?: number; maxStopWords?: number; diff --git a/core/llm/index.ts b/core/llm/index.ts index 5ecfa33ffa..70565b5ee5 100644 --- a/core/llm/index.ts +++ b/core/llm/index.ts @@ -117,7 +117,7 @@ export abstract class BaseLLM implements ILLM { writeLog?: (str: string) => Promise; llmRequestHook?: (model: string, prompt: string) => any; apiKey?: string; - apiKeySecret?: string; + apiKeyLocation?: string; apiBase?: string; cacheBehavior?: CacheBehavior; capabilities?: ModelCapability; @@ -195,7 +195,7 @@ export abstract class BaseLLM implements ILLM { this.writeLog = options.writeLog; this.llmRequestHook = options.llmRequestHook; this.apiKey = options.apiKey; - this.apiKeySecret = options.apiKeySecret; + this.apiKeyLocation = options.apiKeyLocation; this.aiGatewaySlug = options.aiGatewaySlug; this.apiBase = options.apiBase; this.cacheBehavior = options.cacheBehavior; diff --git a/core/llm/llms/stubs/ContinueProxy.ts b/core/llm/llms/stubs/ContinueProxy.ts index 59d05b115a..848b3fff72 100644 --- a/core/llm/llms/stubs/ContinueProxy.ts +++ b/core/llm/llms/stubs/ContinueProxy.ts @@ -14,12 +14,11 @@ class ContinueProxy extends OpenAI { // but we need to keep track of the actual values that the proxy will use // to call whatever LLM API is chosen private actualApiBase?: string; - private actualApiKey?: string; constructor(options: LLMOptions) { super(options); this.actualApiBase = options.apiBase; - this.actualApiKey = options.apiKey; + this.apiKeyLocation = options.apiKeyLocation; } static providerName = "continue-proxy"; @@ -30,9 +29,8 @@ class ContinueProxy extends OpenAI { protected extraBodyProperties(): Record { return { continueProperties: { - apiKey: this.actualApiKey, + apiKeyLocation: this.apiKeyLocation, apiBase: this.actualApiBase, - apiKeySecret: this.apiKeySecret, }, }; } diff --git a/core/package-lock.json b/core/package-lock.json index 350607e4d2..ec82ab0cdf 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -13,7 +13,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", @@ -3038,9 +3038,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.11.tgz", - "integrity": "sha512-E3RBQfNEPBGBmlnAbCXgeAasDzTjo4ON/HH0hr5g292i+WdAN3i/omjQ6Iusx00L1Fz7klZGJePZ3GVQKOGEUg==", + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.16.tgz", + "integrity": "sha512-r3dzTvL8aQ5RrzM+FX6fF+U12/crU4wpPXgDcDloSsxiOQfu3k/shLCbNzFPW57XhLUQAmUJYSZsJbIvYrNPNA==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", diff --git a/core/package.json b/core/package.json index 0e107f9916..477290c9bc 100644 --- a/core/package.json +++ b/core/package.json @@ -46,7 +46,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index 31f06cbb14..38d0db7c80 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.9.252", + "version": "0.9.254", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "continue", - "version": "0.9.252", + "version": "0.9.254", "license": "Apache-2.0", "dependencies": { "@continuedev/fetch": "^1.0.3", @@ -106,7 +106,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index f6520de7d3..23c151699b 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -2,7 +2,7 @@ "name": "continue", "icon": "media/icon.png", "author": "Continue Dev, Inc", - "version": "0.9.253", + "version": "0.9.254", "repository": { "type": "git", "url": "https://github.com/continuedev/continue" diff --git a/gui/package-lock.json b/gui/package-lock.json index d99d5ebab7..922ac8e10c 100644 --- a/gui/package-lock.json +++ b/gui/package-lock.json @@ -7,7 +7,7 @@ "name": "gui", "license": "Apache-2.0", "dependencies": { - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.15", "@headlessui/react": "^1.7.17", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", @@ -108,7 +108,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", @@ -556,9 +556,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.11.tgz", - "integrity": "sha512-E3RBQfNEPBGBmlnAbCXgeAasDzTjo4ON/HH0hr5g292i+WdAN3i/omjQ6Iusx00L1Fz7klZGJePZ3GVQKOGEUg==", + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.15.tgz", + "integrity": "sha512-00c1pz+FqAVEWLo8t71OJzm1KBAAJCXqIjd917Vy8lr7xO0Eoj42lo2GFlVRnqQ0hQvhFQ0s9onHhRag/phHZA==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", @@ -13748,9 +13748,9 @@ } }, "node_modules/zod": { - "version": "3.23.8", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", - "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", + "integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==", "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/gui/package.json b/gui/package.json index bd58519fb0..5bed963ddc 100644 --- a/gui/package.json +++ b/gui/package.json @@ -15,7 +15,7 @@ "test:watch": "vitest" }, "dependencies": { - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.15", "@headlessui/react": "^1.7.17", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", diff --git a/gui/src/components/markdown/FilenameLink.tsx b/gui/src/components/markdown/FilenameLink.tsx index 6ff704cc05..baa0d3a90d 100644 --- a/gui/src/components/markdown/FilenameLink.tsx +++ b/gui/src/components/markdown/FilenameLink.tsx @@ -1,10 +1,10 @@ import { RangeInFile } from "core"; +import { findUriInDirs, getUriPathBasename } from "core/util/uri"; import { useContext } from "react"; +import { v4 as uuidv4 } from "uuid"; import { IdeMessengerContext } from "../../context/IdeMessenger"; import FileIcon from "../FileIcon"; -import { findUriInDirs, getUriPathBasename } from "core/util/uri"; import { ToolTip } from "../gui/Tooltip"; -import { v4 as uuidv4 } from "uuid"; interface FilenameLinkProps { rif: RangeInFile; @@ -23,10 +23,16 @@ function FilenameLink({ rif }: FilenameLinkProps) { const id = uuidv4(); - const { relativePathOrBasename } = findUriInDirs( - rif.filepath, - window.workspacePaths ?? [], - ); + let relPathOrBasename = ""; + try { + const { relativePathOrBasename } = findUriInDirs( + rif.filepath, + window.workspacePaths ?? [], + ); + relPathOrBasename = relativePathOrBasename; + } catch (e) { + return {getUriPathBasename(rif.filepath)}; + } return ( <> @@ -42,7 +48,7 @@ function FilenameLink({ rif }: FilenameLinkProps) { - {"/" + relativePathOrBasename} + {"/" + relPathOrBasename} ); diff --git a/packages/config-yaml/package.json b/packages/config-yaml/package.json index 1e77d7d228..85474ad170 100644 --- a/packages/config-yaml/package.json +++ b/packages/config-yaml/package.json @@ -1,6 +1,6 @@ { "name": "@continuedev/config-yaml", - "version": "1.0.11", + "version": "1.0.16", "description": "", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/config-yaml/src/README.md b/packages/config-yaml/src/README.md new file mode 100644 index 0000000000..4ef80cb49a --- /dev/null +++ b/packages/config-yaml/src/README.md @@ -0,0 +1,17 @@ +# config.yaml specification + +This specification is a work in progress and subject to change. + +## Loading a config.yaml file + +config.yaml is loaded in the following steps + +## Unrolling + +A "source" config.yaml is "unrolled" so that its packages all get merged into a single config.yaml. This is done by recursively loading all packages and merging them into the config.yaml. + +This happens on the server, unless using local mode. + +## Client rendering + +The unrolled config.yaml is then rendered on the client. This is done by replacing all user secret template variables with their values and replacing all other secrets with secret locations. diff --git a/packages/config-yaml/src/converter.ts b/packages/config-yaml/src/converter.ts index 485d469a9e..56d5326d2a 100644 --- a/packages/config-yaml/src/converter.ts +++ b/packages/config-yaml/src/converter.ts @@ -65,7 +65,6 @@ function convertCustomCommand( name: cmd.name, description: cmd.description, prompt: (cmd as any).prompt, // The type is wrong in @continuedev/config-types - type: "slash-command", }; } diff --git a/packages/config-yaml/src/index.ts b/packages/config-yaml/src/index.ts index c3be3e7133..9ebf6cb320 100644 --- a/packages/config-yaml/src/index.ts +++ b/packages/config-yaml/src/index.ts @@ -1,107 +1,11 @@ -import * as YAML from "yaml"; -import { ConfigYaml, configYamlSchema } from "./schemas/index.js"; - -const REGISTRY_URL = "https://registry.continue.dev"; -const LATEST = "latest"; - -function parseUses(uses: string): { - owner: string; - packageName: string; - version: string; -} { - const [owner, packageNameAndVersion] = uses.split("/"); - const [packageName, version] = packageNameAndVersion.split("@"); - return { - owner, - packageName, - version: version ?? LATEST, - }; -} - -export function extendConfig(config: ConfigYaml, pkg: ConfigYaml): ConfigYaml { - return { - ...config, - models: [...(config.models ?? []), ...(pkg.models ?? [])], - context: [...(config.context ?? []), ...(pkg.context ?? [])], - tools: [...(config.tools ?? []), ...(pkg.tools ?? [])], - data: [...(config.data ?? []), ...(pkg.data ?? [])], - mcpServers: [...(config.mcpServers ?? []), ...(pkg.mcpServers ?? [])], - }; -} - -export async function resolvePackages( - configYaml: ConfigYaml, -): Promise { - if (!configYaml.packages) return configYaml; - - for (const pkgDesc of configYaml.packages) { - const { owner, packageName, version } = parseUses(pkgDesc.uses); - const downloadUrl = new URL( - `/${owner}/${packageName}/${version}`, - REGISTRY_URL, - ); - const resp = await fetch(downloadUrl); - if (!resp.ok) { - throw new Error( - `Failed to fetch package ${pkgDesc.uses} from registry: ${resp.statusText}`, - ); - } - const downloadBuf = await resp.arrayBuffer(); - const downloadStr = new TextDecoder().decode(downloadBuf); - const pkg = YAML.parse(downloadStr); - - const validatedPkg = configYamlSchema.parse(pkg); - configYaml = extendConfig(configYaml, validatedPkg); - } - return configYaml; -} - -export function renderConfigYaml(configYaml: string): ConfigYaml { - try { - const parsed = YAML.parse(configYaml); - const result = configYamlSchema.parse(parsed); - return result; - } catch (e: any) { - throw new Error(`Failed to parse config yaml: ${e.message}`); - } -} - -const TEMPLATE_VAR_REGEX = /\${{[\s]*([^}\s]+)[\s]*}}/g; - -export function getTemplateVariables(templatedYaml: string): string[] { - const variables = new Set(); - const matches = templatedYaml.matchAll(TEMPLATE_VAR_REGEX); - for (const match of matches) { - variables.add(match[1]); - } - return Array.from(variables); -} - -export function fillTemplateVariables( - templatedYaml: string, - data: { [key: string]: string }, -): string { - return templatedYaml.replace(TEMPLATE_VAR_REGEX, (match, variableName) => { - // Inject data - if (variableName in data) { - return data[variableName]; - } - // If variable doesn't exist, return the original expression - return match; - }); -} - -export { convertJsonToYamlConfig } from "./converter.js"; -export { resolveSecretsOnClient } from "./resolveSecretsOnClient.js"; -export { - ClientConfigYaml, - clientConfigYamlSchema, - ConfigYaml, - configYamlSchema, -} from "./schemas/index.js"; +export * from "./converter.js"; +export * from "./interfaces/index.js"; +export * from "./interfaces/SecretResult.js"; +export * from "./interfaces/slugs.js"; +export * from "./load/clientRender.js"; +export * from "./load/merge.js"; +export * from "./load/proxySecretResolution.js"; +export * from "./load/unroll.js"; +export * from "./schemas/index.js"; export type { ModelConfig } from "./schemas/models.js"; -export { - ConfigResult, - ConfigValidationError, - validateConfigYaml, -} from "./validation.js"; +export * from "./validation.js"; diff --git a/packages/config-yaml/src/interfaces/SecretResult.test.ts b/packages/config-yaml/src/interfaces/SecretResult.test.ts new file mode 100644 index 0000000000..2828f015cf --- /dev/null +++ b/packages/config-yaml/src/interfaces/SecretResult.test.ts @@ -0,0 +1,41 @@ +import { + SecretType, + decodeSecretLocation, + encodeSecretLocation, +} from "./SecretResult.js"; +import { PackageSlug } from "./slugs.js"; + +describe("SecretLocation encoding/decoding", () => { + it("encodes/decodes organization secret location", () => { + const orgSecretLocation = { + secretType: SecretType.Organization as const, + orgSlug: "test-org", + secretName: "secret1", + }; + + const encoded = encodeSecretLocation(orgSecretLocation); + expect(encoded).toBe("organization:test-org/secret1"); + + const decoded = decodeSecretLocation(encoded); + expect(decoded).toEqual(orgSecretLocation); + }); + + it("encodes/decodes package secret location", () => { + const packageSlug: PackageSlug = { + ownerSlug: "test-org", + packageSlug: "test-package", + }; + + const packageSecretLocation = { + secretType: SecretType.Package as const, + packageSlug, + secretName: "secret1", + }; + + const encoded = encodeSecretLocation(packageSecretLocation); + expect(encoded).toBe("package:test-org/test-package/secret1"); + + const decoded = decodeSecretLocation(encoded); + expect(decoded).toEqual(packageSecretLocation); + }); +}); diff --git a/packages/config-yaml/src/interfaces/SecretResult.ts b/packages/config-yaml/src/interfaces/SecretResult.ts new file mode 100644 index 0000000000..987b336763 --- /dev/null +++ b/packages/config-yaml/src/interfaces/SecretResult.ts @@ -0,0 +1,92 @@ +import { FQSN, PackageSlug, encodePackageSlug } from "./slugs.js"; + +export enum SecretType { + User = "user", + Package = "package", + Organization = "organization", +} + +export interface OrgSecretLocation { + secretType: SecretType.Organization; + orgSlug: string; + secretName: string; +} + +export interface PackageSecretLocation { + secretType: SecretType.Package; + packageSlug: PackageSlug; + secretName: string; +} + +export interface UserSecretLocation { + secretType: SecretType.User; + userSlug: string; + secretName: string; +} + +export type SecretLocation = + | OrgSecretLocation + | PackageSecretLocation + | UserSecretLocation; + +export function encodeSecretLocation(secretLocation: SecretLocation): string { + if (secretLocation.secretType === SecretType.Organization) { + return `${SecretType.Organization}:${secretLocation.orgSlug}/${secretLocation.secretName}`; + } else if (secretLocation.secretType === SecretType.User) { + return `${SecretType.User}:${secretLocation.userSlug}/${secretLocation.secretName}`; + } else { + return `${SecretType.Package}:${encodePackageSlug(secretLocation.packageSlug)}/${secretLocation.secretName}`; + } +} + +export function decodeSecretLocation(secretLocation: string): SecretLocation { + const [secretType, rest] = secretLocation.split(":"); + const parts = rest.split("/"); + const secretName = parts[parts.length - 1]; + + switch (secretType) { + case SecretType.Organization: + return { + secretType: SecretType.Organization, + orgSlug: parts[0], + secretName, + }; + case SecretType.User: + return { + secretType: SecretType.User, + userSlug: parts[0], + secretName, + }; + case SecretType.Package: + return { + secretType: SecretType.Package, + packageSlug: { ownerSlug: parts[0], packageSlug: parts[1] }, + secretName, + }; + default: + throw new Error(`Invalid secret type: ${secretType}`); + } +} + +export interface NotFoundSecretResult { + found: false; + fqsn: FQSN; +} + +export interface FoundSecretResult { + found: true; + secretLocation: OrgSecretLocation | PackageSecretLocation; + fqsn: FQSN; +} + +export interface FoundUserSecretResult { + found: true; + secretLocation: UserSecretLocation; + value: string; + fqsn: FQSN; +} + +export type SecretResult = + | FoundSecretResult + | FoundUserSecretResult + | NotFoundSecretResult; diff --git a/packages/config-yaml/src/interfaces/index.ts b/packages/config-yaml/src/interfaces/index.ts new file mode 100644 index 0000000000..8ad1d9c38f --- /dev/null +++ b/packages/config-yaml/src/interfaces/index.ts @@ -0,0 +1,86 @@ +import { SecretLocation, SecretResult, SecretType } from "./SecretResult.js"; +import { FQSN, FullSlug } from "./slugs.js"; + +/** + * A registry stores the content of packages + */ +export interface Registry { + getContent(fullSlug: FullSlug): Promise; +} +export type SecretNamesMap = Map; + +/** + * A secret store stores secrets + */ +export interface SecretStore { + get(secretName: string): Promise; + set(secretName: string, secretValue: string): Promise; +} + +export interface PlatformClient { + resolveFQSNs(fqsns: FQSN[]): Promise<(SecretResult | undefined)[]>; +} + +export interface PlatformSecretStore { + getSecretFromSecretLocation( + secretLocation: SecretLocation, + ): Promise; +} + +export async function resolveFQSN( + currentUserSlug: string, + fqsn: FQSN, + platformSecretStore: PlatformSecretStore, +): Promise { + // First create the list of secret locations to try in order + const reversedSlugs = [...fqsn.packageSlugs].reverse(); + + const locationsToLook: SecretLocation[] = [ + // Packages first + ...reversedSlugs.map((slug) => ({ + secretType: SecretType.Package as const, + packageSlug: slug, + secretName: fqsn.secretName, + })), + // Then user + { + secretType: SecretType.User as const, + userSlug: currentUserSlug, + secretName: fqsn.secretName, + }, + // Then organization + ...reversedSlugs.map((slug) => ({ + secretType: SecretType.Organization as const, + orgSlug: slug.ownerSlug, + secretName: fqsn.secretName, + })), + ]; + + // Then try to get the secret from each location + for (const secretLocation of locationsToLook) { + const secret = + await platformSecretStore.getSecretFromSecretLocation(secretLocation); + if (secret) { + if (secretLocation.secretType === SecretType.User) { + // Only user secret values get sent back to client + return { + found: true, + fqsn, + secretLocation, + value: secret, + }; + } else { + return { + found: true, + fqsn, + secretLocation, + }; + } + } + } + + return { + found: false, + fqsn, + }; +} diff --git a/packages/config-yaml/src/interfaces/slugs.test.ts b/packages/config-yaml/src/interfaces/slugs.test.ts new file mode 100644 index 0000000000..cab57e5d24 --- /dev/null +++ b/packages/config-yaml/src/interfaces/slugs.test.ts @@ -0,0 +1,82 @@ +import { + decodeFQSN, + decodeFullSlug, + decodePackageSlug, + encodeFQSN, + encodeFullSlug, + encodePackageSlug, + VirtualTags, +} from "./slugs.js"; + +describe("PackageSlug", () => { + it("should encode/decode package slugs", () => { + const testSlug = { + ownerSlug: "test-owner", + packageSlug: "test-package", + }; + const encoded = encodePackageSlug(testSlug); + expect(encoded).toBe("test-owner/test-package"); + const decoded = decodePackageSlug(encoded); + expect(decoded).toEqual(testSlug); + }); + + it("should encode/decode full slugs", () => { + const testFullSlug = { + ownerSlug: "test-owner", + packageSlug: "test-package", + versionSlug: "1.0.0", + }; + const encoded = encodeFullSlug(testFullSlug); + expect(encoded).toBe("test-owner/test-package@1.0.0"); + const decoded = decodeFullSlug(encoded); + expect(decoded).toEqual(testFullSlug); + }); + + it("should use latest tag when no version provided", () => { + const encoded = "test-owner/test-package"; + const decoded = decodeFullSlug(encoded); + expect(decoded.versionSlug).toBe(VirtualTags.Latest); + }); + + it("should encode/decode FQSN with single package", () => { + const testFQSN = { + packageSlugs: [ + { + ownerSlug: "test-owner", + packageSlug: "test-package", + }, + ], + secretName: "test-secret", + }; + const encoded = encodeFQSN(testFQSN); + expect(encoded).toBe("test-owner/test-package/test-secret"); + const decoded = decodeFQSN(encoded); + expect(decoded).toEqual(testFQSN); + }); + + it("should encode/decode FQSN with multiple packages", () => { + const testFQSN = { + packageSlugs: [ + { + ownerSlug: "owner1", + packageSlug: "package1", + }, + { + ownerSlug: "owner2", + packageSlug: "package2", + }, + ], + secretName: "test-secret", + }; + const encoded = encodeFQSN(testFQSN); + expect(encoded).toBe("owner1/package1/owner2/package2/test-secret"); + const decoded = decodeFQSN(encoded); + expect(decoded).toEqual(testFQSN); + }); + + it("should throw error for invalid FQSN format", () => { + expect(() => decodeFQSN("owner1/package1/owner2/test-secret")).toThrow( + "Invalid FQSN format: package slug must have two parts", + ); + }); +}); diff --git a/packages/config-yaml/src/interfaces/slugs.ts b/packages/config-yaml/src/interfaces/slugs.ts new file mode 100644 index 0000000000..a6926fb5b6 --- /dev/null +++ b/packages/config-yaml/src/interfaces/slugs.ts @@ -0,0 +1,68 @@ +export interface PackageSlug { + ownerSlug: string; + packageSlug: string; +} +export interface FullSlug extends PackageSlug { + versionSlug: string; +} + +export enum VirtualTags { + Latest = "latest", +} + +export function encodePackageSlug(packageSlug: PackageSlug): string { + return `${packageSlug.ownerSlug}/${packageSlug.packageSlug}`; +} + +export function decodePackageSlug(pkgSlug: string): PackageSlug { + const [ownerSlug, packageSlug] = pkgSlug.split("/"); + return { + ownerSlug, + packageSlug, + }; +} + +export function encodeFullSlug(fullSlug: FullSlug): string { + return `${fullSlug.ownerSlug}/${fullSlug.packageSlug}@${fullSlug.versionSlug}`; +} + +export function decodeFullSlug(fullSlug: string): FullSlug { + const [ownerSlug, packageSlug, versionSlug] = fullSlug.split(/[/@]/); + return { + ownerSlug, + packageSlug, + versionSlug: versionSlug || VirtualTags.Latest, + }; +} + +/** + * FQSN = Fully Qualified Secret Name + */ +export interface FQSN { + packageSlugs: PackageSlug[]; + secretName: string; +} + +export function encodeFQSN(fqsn: FQSN): string { + const parts = [...fqsn.packageSlugs.map(encodePackageSlug), fqsn.secretName]; + return parts.join("/"); +} + +export function decodeFQSN(fqsn: string): FQSN { + const parts = fqsn.split("/"); + const secretName = parts.pop()!; + const packageSlugs: PackageSlug[] = []; + + // Process parts two at a time to decode package slugs + for (let i = 0; i < parts.length; i += 2) { + if (i + 1 >= parts.length) { + throw new Error("Invalid FQSN format: package slug must have two parts"); + } + packageSlugs.push({ + ownerSlug: parts[i], + packageSlug: parts[i + 1], + }); + } + + return { packageSlugs, secretName }; +} diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts new file mode 100644 index 0000000000..cf9b6ba918 --- /dev/null +++ b/packages/config-yaml/src/load/clientRender.ts @@ -0,0 +1,123 @@ +import { PlatformClient, SecretStore } from "../interfaces/index.js"; +import { + decodeSecretLocation, + encodeSecretLocation, + SecretLocation, +} from "../interfaces/SecretResult.js"; +import { decodeFQSN, encodeFQSN, FQSN } from "../interfaces/slugs.js"; +import { ConfigYaml } from "../schemas/index.js"; +import { + fillTemplateVariables, + getTemplateVariables, + parseConfigYaml, +} from "./unroll.js"; + +export async function clientRender( + unrolledConfigContent: string, + secretStore: SecretStore, + platformClient?: PlatformClient, +): Promise { + // 1. First we need to get a list of all the FQSNs that are required to render the config + const secrets = getTemplateVariables(unrolledConfigContent); + + // 2. Then, we will check which of the secrets are found in the local personal secret store. Here we’re checking for anything that matches the last part of the FQSN, not worrying about the owner/package/owner/package slugs + const secretsTemplateData: Record = {}; + + const unresolvedFQSNs: FQSN[] = []; + for (const secret of secrets) { + const fqsn = decodeFQSN(secret.replace("secrets.", "")); + const secretValue = await secretStore.get(fqsn.secretName); + if (secretValue) { + secretsTemplateData[secret] = secretValue; + } else { + unresolvedFQSNs.push(fqsn); + } + } + + // Don't use platform client in local mode + if (platformClient) { + // 3. For any secrets not found, we send the FQSNs to the Continue Platform at the `/ide/sync-secrets` endpoint. This endpoint replies for each of the FQSNs with the following information (`SecretResult`): `foundAt`: tells which secret store it was found in (this is “user”, “org”, “package” or null if not found anywhere). If it’s found in an org or a package, it tells us the `secretLocation`, which is either just an org slug, or is a full org/package slug. If it’s found in “user” secrets, we send back the `value`. Full definition of `SecretResult` at [2]. The method of resolving an FQSN to a `SecretResult` is detailed at [3] + const secretResults = await platformClient.resolveFQSNs(unresolvedFQSNs); + + // 4. (back to the client) Any “user” secrets that were returned back are added to the local secret store so we don’t have to request them again + for (const secretResult of secretResults) { + if (!secretResult) { + continue; + } + + if (!secretResult.found) { + // When a secret isn't found anywhere, we keep it templated as just the secret name + // in case it can be found in the on-prem proxy's env + secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = + `\${{ secrets.${secretResult.fqsn.secretName} }}`; + continue; + } + + if ("value" in secretResult) { + secretStore.set(secretResult.fqsn.secretName, secretResult.value); + } + + secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = + "value" in secretResult + ? secretResult.value + : `\${{ secrets.${encodeSecretLocation(secretResult.secretLocation)} }}`; + } + } + + // 5. User secrets are rendered in place of the template variable. Others remain templated, but replaced with the specific location where they are to be found (`${{ secrets. }}` instead of `${{ secrets. }}`) + const renderedYaml = fillTemplateVariables( + unrolledConfigContent, + secretsTemplateData, + ); + + // 6. The rendered YAML is parsed and validated again + const parsedYaml = parseConfigYaml(renderedYaml); + + // 7. We update any of the items with the proxy version if there are un-rendered secrets + const finalConfig = useProxyForUnrenderedSecrets(parsedYaml); + return finalConfig; +} + +function getUnrenderedSecretLocation( + value: string | undefined, +): SecretLocation | undefined { + if (!value) return undefined; + + const templateVars = getTemplateVariables(value); + if (templateVars.length === 1) { + const secretLocationEncoded = templateVars[0].split("secrets.")[1]; + try { + const secretLocation = decodeSecretLocation(secretLocationEncoded); + return secretLocation; + } catch (e) { + // If it's a templated secret but not a valid secret location, leave it be + // in case on-prem proxy has the secret in an env variable + if (templateVars[0].startsWith("secrets.")) { + return undefined; // TODO + } + return undefined; + } + } + + return undefined; +} + +function useProxyForUnrenderedSecrets(config: ConfigYaml): ConfigYaml { + if (config.models) { + for (let i = 0; i < config.models.length; i++) { + const apiKeyLocation = getUnrenderedSecretLocation( + config.models[i].apiKey, + ); + if (apiKeyLocation) { + config.models[i] = { + ...config.models[i], + provider: "continue-proxy", + apiKeyLocation: encodeSecretLocation(apiKeyLocation), + apiKey: undefined, + }; + } + } + } + + return config; +} diff --git a/packages/config-yaml/src/load/merge.ts b/packages/config-yaml/src/load/merge.ts new file mode 100644 index 0000000000..590572edba --- /dev/null +++ b/packages/config-yaml/src/load/merge.ts @@ -0,0 +1,18 @@ +import { ConfigYaml } from "../schemas/index.js"; + +export function mergePackages( + current: ConfigYaml, + incoming: ConfigYaml, +): ConfigYaml { + return { + ...current, + models: [...(current.models ?? []), ...(incoming.models ?? [])], + context: [...(current.context ?? []), ...(incoming.context ?? [])], + data: [...(current.data ?? []), ...(incoming.data ?? [])], + tools: [...(current.tools ?? []), ...(incoming.tools ?? [])], + mcpServers: [...(current.mcpServers ?? []), ...(incoming.mcpServers ?? [])], + rules: [...(current.rules ?? []), ...(incoming.rules ?? [])], + prompts: [...(current.prompts ?? []), ...(incoming.prompts ?? [])], + docs: [...(current.docs ?? []), ...(incoming.docs ?? [])], + }; +} diff --git a/packages/config-yaml/src/load/proxySecretResolution.ts b/packages/config-yaml/src/load/proxySecretResolution.ts new file mode 100644 index 0000000000..ff3b0ec4e6 --- /dev/null +++ b/packages/config-yaml/src/load/proxySecretResolution.ts @@ -0,0 +1,32 @@ +import { PlatformSecretStore, SecretStore } from "../interfaces/index.js"; +import { + SecretLocation, + encodeSecretLocation, +} from "../interfaces/SecretResult.js"; + +export async function resolveSecretLocationInProxy( + secretLocaton: SecretLocation, + platformSecretStore: PlatformSecretStore, + environmentSecretStore?: SecretStore, +): Promise { + // 1. Check environment variables (if supported) + if (environmentSecretStore) { + const envSecretValue = await environmentSecretStore.get( + secretLocaton.secretName, + ); + if (envSecretValue) { + return envSecretValue; + } + } + + // 2. Get from secret location + const platformSecret = + await platformSecretStore.getSecretFromSecretLocation(secretLocaton); + if (platformSecret) { + return platformSecret; + } + + throw new Error( + `Could not resolve secret with location ${encodeSecretLocation(secretLocaton)}`, + ); +} diff --git a/packages/config-yaml/src/load/unroll.ts b/packages/config-yaml/src/load/unroll.ts new file mode 100644 index 0000000000..54774c5a3e --- /dev/null +++ b/packages/config-yaml/src/load/unroll.ts @@ -0,0 +1,172 @@ +import * as YAML from "yaml"; +import { Registry } from "../interfaces/index.js"; +import { + PackageSlug, + decodeFullSlug, + encodePackageSlug, +} from "../interfaces/slugs.js"; +import { ConfigYaml, configYamlSchema } from "../schemas/index.js"; +import { mergePackages } from "./merge.js"; + +export function parseConfigYaml(configYaml: string): ConfigYaml { + try { + const parsed = YAML.parse(configYaml); + const result = configYamlSchema.parse(parsed); + return result; + } catch (e: any) { + throw new Error(`Failed to parse config yaml: ${e.message}`); + } +} + +const TEMPLATE_VAR_REGEX = /\${{[\s]*([^}\s]+)[\s]*}}/g; + +export function getTemplateVariables(templatedYaml: string): string[] { + const variables = new Set(); + const matches = templatedYaml.matchAll(TEMPLATE_VAR_REGEX); + for (const match of matches) { + variables.add(match[1]); + } + return Array.from(variables); +} + +export function fillTemplateVariables( + templatedYaml: string, + data: { [key: string]: string }, +): string { + return templatedYaml.replace(TEMPLATE_VAR_REGEX, (match, variableName) => { + // Inject data + if (variableName in data) { + return data[variableName]; + } + // If variable doesn't exist, return the original expression + return match; + }); +} + +export async function unrollImportedPackage( + pkgImport: NonNullable[number], + parentPackages: PackageSlug[], + registry: Registry, +): Promise { + const { uses, with: params } = pkgImport; + + const fullSlug = decodeFullSlug(uses); + + // Request the content from the registry + const rawContent = await registry.getContent(fullSlug); + + // Convert the raw YAML to unrolled config + return await unrollPackageFromContent( + rawContent, + params, + parentPackages, + registry, + ); +} + +export interface TemplateData { + params: Record | undefined; + secrets: Record | undefined; + continue: {}; +} + +function flattenTemplateData( + templateData: TemplateData, +): Record { + const flattened: Record = {}; + + if (templateData.params) { + for (const [key, value] of Object.entries(templateData.params)) { + flattened[`params.${key}`] = value; + } + } + if (templateData.secrets) { + for (const [key, value] of Object.entries(templateData.secrets)) { + flattened[`secrets.${key}`] = value; + } + } + + return flattened; +} + +function secretToFQSNMap( + secretNames: string[], + parentPackages: PackageSlug[], +): Record { + const map: Record = {}; + for (const secret of secretNames) { + const parentSlugs = parentPackages.map(encodePackageSlug); + const parts = [...parentSlugs, secret]; + const fqsn = parts.join("/"); + map[secret] = `\${{ secrets.${fqsn} }}`; + } + + return map; +} + +function extractFQSNMap( + rawContent: string, + parentPackages: PackageSlug[], +): Record { + const templateVars = getTemplateVariables(rawContent); + const secrets = templateVars + .filter((v) => v.startsWith("secrets.")) + .map((v) => v.replace("secrets.", "")); + + return secretToFQSNMap(secrets, parentPackages); +} + +export async function unrollPackageFromContent( + rawContent: string, + params: Record | undefined, + packagePath: PackageSlug[], + registry: Registry, +): Promise { + // Collect template data + const templateData: TemplateData = { + // params are passed from the parent package + params: params, + // at this stage, secrets are mapped to a (still templated) FQSN + secrets: extractFQSNMap(rawContent, packagePath), + // Built-in variables + continue: {}, + }; + + const templatedYaml = fillTemplateVariables( + rawContent, + flattenTemplateData(templateData), + ); + + let parsedYaml = parseConfigYaml(templatedYaml); + + const unrolledChildPackages = await Promise.all( + parsedYaml.packages?.map((pkg) => { + const pkgSlug = decodeFullSlug(pkg.uses); + return unrollImportedPackage(pkg, [...packagePath, pkgSlug], registry); + }) ?? [], + ); + + delete parsedYaml.packages; + for (const childPkg of unrolledChildPackages) { + parsedYaml = mergePackages(parsedYaml, childPkg); + } + + return parsedYaml; +} + +/** + * Loading an assistant is equivalent to loading a package without params + */ +export async function unrollAssistant( + fullSlug: string, + registry: Registry, +): Promise { + const packageSlug = decodeFullSlug(fullSlug); + return await unrollImportedPackage( + { + uses: fullSlug, + }, + [packageSlug], + registry, + ); +} diff --git a/packages/config-yaml/src/resolveSecretsOnClient.ts b/packages/config-yaml/src/resolveSecretsOnClient.ts deleted file mode 100644 index c3029fbda2..0000000000 --- a/packages/config-yaml/src/resolveSecretsOnClient.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { ClientConfigYaml, ConfigYaml } from "./schemas/index.js"; -type SecretProvider = ( - secretNames: string[], -) => Promise<{ [key: string]: string }>; - -/** - * Take a ConfigYaml with apiKeySecrets, and look to fill in these secrets - * with whatever secret store exists in the client. - */ -export async function resolveSecretsOnClient( - configYaml: ClientConfigYaml, - getSecretsFromClientStore: SecretProvider, - getSecretsFromServer: SecretProvider, -): Promise { - const requiredSecrets = getRequiredSecretsInClientConfig(configYaml); - - const secretsFoundOnClient = await getSecretsFromClientStore(requiredSecrets); - - const secretsNotFoundOnClient = requiredSecrets.filter( - (secret) => !secretsFoundOnClient[secret], - ); - - let secretsFoundOnServer = {}; - if (secretsNotFoundOnClient.length > 0) { - secretsFoundOnServer = await getSecretsFromServer(secretsNotFoundOnClient); - } - - const clientSecrets = { - ...secretsFoundOnClient, - ...secretsFoundOnServer, - }; - - const finalConfigYaml = injectClientSecrets(configYaml, clientSecrets); - - // Anything with an apiKeySecret left over must use proxy - return finalConfigYaml; -} - -function getRequiredSecretsInClientConfig( - configYaml: ClientConfigYaml, -): string[] { - const secrets = new Set(); - for (const model of configYaml.models ?? []) { - if (model.apiKeySecret) { - secrets.add(model.apiKeySecret); - } - } - return Array.from(secrets); -} - -function injectClientSecrets( - configYaml: ClientConfigYaml, - clientSecrets: Record, -): ConfigYaml { - for (const model of configYaml.models ?? []) { - if (model.apiKeySecret && clientSecrets[model.apiKeySecret]) { - // Remove apiKeySecret and place the client secret in apiKey - model.apiKey = clientSecrets[model.apiKeySecret]; - delete model.apiKeySecret; - } - } - - return configYaml; -} diff --git a/packages/config-yaml/src/schemas/context.ts b/packages/config-yaml/src/schemas/context.ts deleted file mode 100644 index 4441bd5a87..0000000000 --- a/packages/config-yaml/src/schemas/context.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { z } from "zod"; - -export const contextSchema = z.object({ - uses: z.string(), - with: z.any().optional(), -}); diff --git a/packages/config-yaml/src/schemas/data.ts b/packages/config-yaml/src/schemas/data.ts deleted file mode 100644 index 573da2448a..0000000000 --- a/packages/config-yaml/src/schemas/data.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { z } from "zod"; - -export const dataSchema = z.object({ - provider: z.string(), -}); diff --git a/packages/config-yaml/src/schemas/index.ts b/packages/config-yaml/src/schemas/index.ts index f763c7be56..394c1dd410 100644 --- a/packages/config-yaml/src/schemas/index.ts +++ b/packages/config-yaml/src/schemas/index.ts @@ -1,18 +1,23 @@ import * as z from "zod"; -import { contextSchema } from "./context.js"; -import { dataSchema } from "./data.js"; import { modelSchema } from "./models.js"; const packageSchema = z.object({ uses: z.string(), with: z.any().optional(), - secrets: z.array(z.string()).optional(), +}); + +export const dataSchema = z.object({ + provider: z.string(), +}); + +export const contextSchema = z.object({ + uses: z.string(), + with: z.any().optional(), }); const toolSchema = z.object({ name: z.string(), description: z.string(), - policy: z.enum(["automatic", "allowed", "disabled"]).optional(), url: z.string(), apiKey: z.string().optional(), }); @@ -21,14 +26,12 @@ const mcpServerSchema = z.object({ name: z.string(), command: z.string(), args: z.array(z.string()).optional(), - env: z.record(z.string()).optional(), }); const promptSchema = z.object({ name: z.string(), description: z.string().optional(), - type: z.enum(["slash-command", "context-provider"]).optional(), prompt: z.string(), }); @@ -55,6 +58,8 @@ export const configYamlSchema = z.object({ export type ConfigYaml = z.infer; -export const clientConfigYamlSchema = configYamlSchema.omit({ packages: true }); +export const unrolledConfigYamlSchema = configYamlSchema.omit({ + packages: true, +}); -export type ClientConfigYaml = z.infer; +export type UnrolledConfigYaml = z.infer; diff --git a/packages/config-yaml/src/schemas/models.ts b/packages/config-yaml/src/schemas/models.ts index 89b740e573..713d2cc2d9 100644 --- a/packages/config-yaml/src/schemas/models.ts +++ b/packages/config-yaml/src/schemas/models.ts @@ -42,16 +42,26 @@ export const completionOptionsSchema = z.object({ }); export type CompletionOptions = z.infer; -export const modelSchema = z.object({ +const baseModelFields = { name: z.string(), - provider: z.string(), model: z.string(), apiKey: z.string().optional(), apiBase: z.string().optional(), - apiKeySecret: z.string().optional(), roles: modelRolesSchema.array().optional(), defaultCompletionOptions: completionOptionsSchema.optional(), requestOptions: requestOptionsSchema.optional(), -}); +}; + +export const modelSchema = z.union([ + z.object({ + ...baseModelFields, + provider: z.literal("continue-proxy"), + apiKeyLocation: z.string(), + }), + z.object({ + ...baseModelFields, + provider: z.string().refine((val) => val !== "continue-proxy"), + }), +]); export type ModelConfig = z.infer; diff --git a/packages/config-yaml/test/index.test.ts b/packages/config-yaml/test/index.test.ts new file mode 100644 index 0000000000..3f75ff0922 --- /dev/null +++ b/packages/config-yaml/test/index.test.ts @@ -0,0 +1,178 @@ +import * as fs from "fs"; +import * as YAML from "yaml"; +import { + decodeSecretLocation, + encodeSecretLocation, + resolveSecretLocationInProxy, +} from "../dist"; +import { + clientRender, + FQSN, + FullSlug, + PlatformClient, + PlatformSecretStore, + Registry, + resolveFQSN, + SecretLocation, + SecretResult, + SecretStore, + SecretType, + unrollAssistant, +} from "../src"; +import exp = require("constants"); + +// Test e2e flows from raw yaml -> unroll -> client render -> resolve secrets on proxy +describe("E2E Scenarios", () => { + const userSecrets: Record = { + OPENAI_API_KEY: "sk-123", + }; + + const packageSecrets: Record = { + "test-org/assistant/ANTHROPIC_API_KEY": "sk-ant", + "test-org/models/GEMINI_API_KEY": "gemini-api-key", + }; + + const proxyEnvSecrets: Record = { + ANTHROPIC_API_KEY: "sk-ant-env", + GEMINI_API_KEY: "gemini-api-key-env", + }; + + const localUserSecretStore: SecretStore = { + get: async function (secretName: string): Promise { + return userSecrets[secretName]; + }, + set: function (secretName: string, secretValue: string): Promise { + throw new Error("Function not implemented."); + }, + }; + + const platformClient: PlatformClient = { + resolveFQSNs: async function ( + fqsns: FQSN[], + ): Promise<(SecretResult | undefined)[]> { + return await Promise.all( + fqsns.map((fqsn) => + resolveFQSN("test-user", fqsn, platformSecretStore), + ), + ); + }, + }; + + const environmentSecretStore: SecretStore = { + get: async function (secretName: string): Promise { + return proxyEnvSecrets[secretName]; + }, + set: function (secretName: string, secretValue: string): Promise { + throw new Error("Function not implemented."); + }, + }; + + const platformSecretStore: PlatformSecretStore = { + getSecretFromSecretLocation: async function ( + secretLocation: SecretLocation, + ): Promise { + if (secretLocation.secretType === SecretType.Package) { + return packageSecrets[ + encodeSecretLocation(secretLocation).split(":")[1] + ]; + } else if (secretLocation.secretType === SecretType.User) { + return userSecrets[secretLocation.secretName]; + } else { + return undefined; + } + }, + }; + + const registry: Registry = { + getContent: async function (fullSlug: FullSlug): Promise { + return fs + .readFileSync( + `./test/packages/${fullSlug.ownerSlug}/${fullSlug.packageSlug}.yaml`, + ) + .toString(); + }, + }; + + it("should correctly unroll assistant", async () => { + const unrolledConfig = await unrollAssistant( + "test-org/assistant", + registry, + ); + + // Test that packages were correctly unrolled and params replaced + expect(unrolledConfig.models?.length).toBe(4); + expect(unrolledConfig.models?.[0].apiKey).toBe( + "${{ secrets.test-org/assistant/OPENAI_API_KEY }}", + ); + expect(unrolledConfig.models?.[1].apiKey).toBe("sk-456"); + expect(unrolledConfig.models?.[2].apiKey).toBe( + "${{ secrets.test-org/assistant/test-org/models/ANTHROPIC_API_KEY }}", + ); + expect(unrolledConfig.models?.[3].apiKey).toBe( + "${{ secrets.test-org/assistant/test-org/models/GEMINI_API_KEY }}", + ); + + expect(unrolledConfig.rules?.length).toBe(3); + expect(unrolledConfig.docs?.[0].startUrl).toBe( + "https://docs.python.org/release/3.13.1", + ); + + const clientRendered = await clientRender( + YAML.stringify(unrolledConfig), + localUserSecretStore, + platformClient, + ); + + // Test that user secrets were injected and others were changed to use proxy + const anthropicSecretLocation = + "package:test-org/assistant/ANTHROPIC_API_KEY"; + const geminiSecretLocation = "package:test-org/models/GEMINI_API_KEY"; + expect(clientRendered.models?.[0].apiKey).toBe("sk-123"); + expect(clientRendered.models?.[1].apiKey).toBe("sk-456"); + expect(clientRendered.models?.[2].provider).toBe("continue-proxy"); + expect((clientRendered.models?.[2] as any).apiKeyLocation).toBe( + anthropicSecretLocation, + ); + expect(clientRendered.models?.[2].apiKey).toBeUndefined(); + expect(clientRendered.models?.[3].provider).toBe("continue-proxy"); + expect((clientRendered.models?.[3] as any).apiKeyLocation).toBe( + geminiSecretLocation, + ); + expect(clientRendered.models?.[3].apiKey).toBeUndefined(); + + // Test that proxy can correctly resolve secrets + const decodedAnthropicSecretLocation = decodeSecretLocation( + anthropicSecretLocation, + ); + const decodedGeminiSecretLocation = + decodeSecretLocation(geminiSecretLocation); + + // With environment + const antSecretValue = await resolveSecretLocationInProxy( + decodedAnthropicSecretLocation, + platformSecretStore, + environmentSecretStore, + ); + expect(antSecretValue).toBe("sk-ant-env"); + const geminiSecretValue = await resolveSecretLocationInProxy( + decodedGeminiSecretLocation, + platformSecretStore, + environmentSecretStore, + ); + expect(geminiSecretValue).toBe("gemini-api-key-env"); + + // Without environment + const antSecretValue2 = await resolveSecretLocationInProxy( + decodedAnthropicSecretLocation, + platformSecretStore, + undefined, + ); + expect(antSecretValue2).toBe("sk-ant"); + const geminiSecretValue2 = await resolveSecretLocationInProxy( + decodedGeminiSecretLocation, + platformSecretStore, + undefined, + ); + expect(geminiSecretValue2).toBe("gemini-api-key"); + }); +}); diff --git a/packages/config-yaml/test/packages/test-org/assistant.yaml b/packages/config-yaml/test/packages/test-org/assistant.yaml new file mode 100644 index 0000000000..74fefc5815 --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/assistant.yaml @@ -0,0 +1,18 @@ +name: Assistant +version: 0.0.1 + +packages: + - uses: test-org/models + - uses: test-org/docs + with: + version: 3.13.1 + - uses: test-org/rules + +models: + - name: gpt-5 + provider: openai + model: gpt-5 + apiKey: ${{ secrets.OPENAI_API_KEY }} + +rules: + - Use KaTeX for math diff --git a/packages/config-yaml/test/packages/test-org/docs.yaml b/packages/config-yaml/test/packages/test-org/docs.yaml new file mode 100644 index 0000000000..1b540d2275 --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/docs.yaml @@ -0,0 +1,7 @@ +name: Docs +version: 0.0.1 + +docs: + - name: Python + startUrl: https://docs.python.org/release/${{ params.version }} + rootUrl: https://docs.python.org/release/${{ params.version }} diff --git a/packages/config-yaml/test/packages/test-org/models.yaml b/packages/config-yaml/test/packages/test-org/models.yaml new file mode 100644 index 0000000000..f47831662d --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/models.yaml @@ -0,0 +1,18 @@ +name: Models +version: 0.0.1 + +models: + - name: gpt-4 + provider: openai + model: gpt-4 + apiKey: sk-456 + + - name: claude-3-5-sonnet-latest + provider: anthropic + model: claude-3-5-sonnet-latest + apiKey: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: gemini + provider: gemini + model: gemini + apiKey: ${{ secrets.GEMINI_API_KEY }} diff --git a/packages/config-yaml/test/packages/test-org/rules.yaml b/packages/config-yaml/test/packages/test-org/rules.yaml new file mode 100644 index 0000000000..2a4f36a9e1 --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/rules.yaml @@ -0,0 +1,6 @@ +name: Rules +version: 0.0.1 + +rules: + - Be kind + - Be concise