Merge pull request #3532 from vercelInfrAI/main

Add new LLM API Provider: Novita AI

CONTRIBUTING.md

@@ -188,7 +188,7 @@ After you've written your context provider, make sure to complete the following:
 
 ### Adding an LLM Provider
 
-Continue has support for more than a dozen different LLM "providers", making it easy to use models running on OpenAI, Ollama, Together, LM Studio, Msty, and more. You can find all of the existing providers [here](https://github.com/continuedev/continue/tree/main/core/llm/llms), and if you see one missing, you can add it with the following steps:
+Continue has support for more than a dozen different LLM "providers", making it easy to use models running on OpenAI, Ollama, Together, Novita AI, LM Studio, Msty, and more. You can find all of the existing providers [here](https://github.com/continuedev/continue/tree/main/core/llm/llms), and if you see one missing, you can add it with the following steps:
 
 1. Create a new file in the `core/llm/llms` directory. The name of the file should be the name of the provider, and it should export a class that extends `BaseLLM`. This class should contain the following minimal implementation. We recommend viewing pre-existing providers for more details. The [LlamaCpp Provider](./core/llm/llms/LlamaCpp.ts) is a good simple example.
 
@@ -209,7 +209,7 @@ While any model that works with a supported provider can be used with Continue,
   1. Add a `ModelPackage` entry for the model into [configs/models.ts](./gui/src/pages/AddNewModel/configs/models.ts), following the lead of the many examples near the top of the file
   2. Add the model within its provider's array to [AddNewModel.tsx](./gui/src/pages/AddNewModel/AddNewModel.tsx) (add provider if needed)
 - [index.d.ts](./core/index.d.ts) - This file defines the TypeScript types used throughout Continue. You'll find a `ModelName` type. Be sure to add the name of your model to this.
-- LLM Providers: Since many providers use their own custom strings to identify models, you'll have to add the translation from Continue's model name (the one you added to `index.d.ts`) and the model string for each of these providers: [Ollama](./core/llm/llms/Ollama.ts), [Together](./core/llm/llms/Together.ts), and [Replicate](./core/llm/llms/Replicate.ts). You can find their full model lists here: [Ollama](https://ollama.ai/library), [Together](https://docs.together.ai/docs/inference-models), [Replicate](https://replicate.com/collections/streaming-language-models).
+- LLM Providers: Since many providers use their own custom strings to identify models, you'll have to add the translation from Continue's model name (the one you added to `index.d.ts`) and the model string for each of these providers: [Ollama](./core/llm/llms/Ollama.ts), [Together](./core/llm/llms/Together.ts), [Novita AI](./core/llm/llms/Novita.ts), and [Replicate](./core/llm/llms/Replicate.ts). You can find their full model lists here: [Ollama](https://ollama.ai/library), [Together](https://docs.together.ai/docs/inference-models), [Novita AI](https://novita.ai/llm-api?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link), [Replicate](https://replicate.com/collections/streaming-language-models).
 - [Prompt Templates](./core/llm/index.ts) - In this file you'll find the `autodetectTemplateType` function. Make sure that for the model name you just added, this function returns the correct template type. This is assuming that the chat template for that model is already built in Continue. If not, you will have to add the template type and corresponding edit and chat templates.
 
 ### Adding Pre-indexed Documentation

core/llm/autodetect.ts

@@ -41,6 +41,7 @@ const PROVIDER_HANDLES_TEMPLATING: string[] = [
   "openai",
   "ollama",
   "together",
+  "novita",
   "msty",
   "anthropic",
   "bedrock",
@@ -130,6 +131,7 @@ const PARALLEL_PROVIDERS: string[] = [
   "free-trial",
   "replicate",
   "together",
+  "novita",
   "sambanova",
   "nebius",
   "vertexai",

core/llm/llms/Novita.ts

@@ -0,0 +1,44 @@
+import OpenAI from "./OpenAI.js";
+
+import type { CompletionOptions, LLMOptions } from "../../index.js";
+
+class Novita extends OpenAI {
+  static providerName = "novita";
+  static defaultOptions: Partial<LLMOptions> = {
+    apiBase: "https://api.novita.ai/v3/openai/",
+  };
+
+  private static MODEL_IDS: { [name: string]: string } = {
+    "llama3-8b": "meta-llama/llama-3-8b-instruct",
+    "llama3-70b": "meta-llama/llama-3-70b-instruct",
+    "llama3.1-8b": "meta-llama/llama-3.1-8b-instruct",
+    "llama3.1-70b": "meta-llama/llama-3.1-70b-instruct",
+    "llama3.1-405b": "meta-llama/llama-3.1-405b-instruct",
+    "llama3.2-1b": "meta-llama/llama-3.2-1b-instruct",
+    "llama3.2-3b": "meta-llama/llama-3.2-3b-instruct",
+    "llama3.2-11b": "meta-llama/llama-3.2-11b-vision-instruct",
+    "llama3.3-70b": "meta-llama/llama-3.3-70b-instruct",
+    "mistral-nemo": "mistralai/mistral-nemo",
+    "mistral-7b": "mistralai/mistral-7b-instruct",
+  };
+
+  protected _convertModelName(model: string) {
+    return Novita.MODEL_IDS[model] || this.model;
+  }
+
+  protected async *_streamComplete(
+    prompt: string,
+    signal: AbortSignal,
+    options: CompletionOptions,
+  ): AsyncGenerator<string> {
+    for await (const chunk of this._legacystreamComplete(
+      prompt,
+      signal,
+      options,
+    )) {
+      yield chunk;
+    }
+  }
+}
+
+export default Novita;

core/llm/llms/index.ts

@@ -48,6 +48,7 @@ import ContinueProxy from "./stubs/ContinueProxy";
 import TestLLM from "./Test";
 import TextGenWebUI from "./TextGenWebUI";
 import Together from "./Together";
+import Novita from "./Novita";
 import VertexAI from "./VertexAI";
 import Vllm from "./Vllm";
 import WatsonX from "./WatsonX";
@@ -65,6 +66,7 @@ export const LLMClasses = [
   Replicate,
   TextGenWebUI,
   Together,
+  Novita,
   HuggingFaceTGI,
   HuggingFaceInferenceAPI,
   Kindo,

docs/docs/chat/model-setup.mdx

@@ -33,7 +33,7 @@ Our current top recommendation is Claude Sonnet 3.5 from [Anthropic](../customiz
 
 ### Llama 3.1 405B from Meta
 
-If you prefer to use an open-weight model, then Llama 3.1 405B from Meta is your best option right now. You will need to decide if you use it through a SaaS model provider (e.g. [Together](../customize/model-providers/more/together.md) or [Groq](../customize/model-providers/more/groq.md)) or self-host it (e.g. using [vLLM](../customize/model-providers//more/vllm.md) or [Ollama](../customize/model-providers/top-level/ollama.md)).
+If you prefer to use an open-weight model, then Llama 3.1 405B from Meta is your best option right now. You will need to decide if you use it through a SaaS model provider (e.g. [Together](../customize/model-providers/more/together.md) or [Novita AI](../customize/model-providers/more/novita.md) or [Groq](../customize/model-providers/more/groq.md)) or self-host it (e.g. using [vLLM](../customize/model-providers//more/vllm.md) or [Ollama](../customize/model-providers/top-level/ollama.md)).
 
 <Tabs groupId="providers">
     <TabItem value="Together">
@@ -48,6 +48,18 @@ If you prefer to use an open-weight model, then Llama 3.1 405B from Meta is your
             ]
         ```
     </TabItem>
+    <TabItem value="Novita">
+        ```json title="config.json"
+            "models": [
+                {
+                    "title": "Llama 3.1 405B",
+                    "provider": "novita",
+                    "model": "meta-llama/llama-3.1-405b-instruct",
+                    "apiKey": "[NOVITA_API_KEY]"
+                }
+            ]
+        ```
+    </TabItem>
     <TabItem value="Groq">
         ```json title="config.json"
             "models": [

docs/docs/customize/model-providers/more/novita.md

@@ -0,0 +1,18 @@
+# Novita
+
+[Novita AI](https://novita.ai?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) offers an affordable, reliable, and simple inference platform with scalable [LLM API](https://novita.ai/docs/model-api/reference/introduction.html), empowering developers to build AI applications. Try the [Novita AI Llama 3 API Demo](https://novita.ai/model-api/product/llm-api/playground/meta-llama-llama-3.1-70b-instruct?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) today!. You can sign up [here](https://novita.ai/user/login?&redirect=/&utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link), copy your API key on the [Key Management](https://novita.ai/settings/key-management?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link), and then hit the play button on any model from the [Novita AI Models list](https://novita.ai/llm-api?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link). Change `~/.continue/config.json` to look like this:
+
+```json title="config.json"
+{
+  "models": [
+    {
+      "title": "Llama 3.1 8B",
+      "provider": "novita",
+      "model": "meta-llama/llama-3.1-8b-instruct",
+      "apiKey": "YOUR_API_KEY"
+    }
+  ]
+}
+```
+
+[View the source](https://github.com/continuedev/continue/blob/main/core/llm/llms/Novita.ts)

docs/docs/customize/tutorials/llama3.1.md

@@ -1,7 +1,7 @@
 ---
 title: Using Llama 3.1 with Continue
 description: How to use Llama 3.1 with Continue
-keywords: [llama, meta, togetherai, ollama, replicate]
+keywords: [llama, meta, togetherai, novita, ollama, replicate]
 ---
 
 Continue makes it easy to code with the latest open-source models, including the entire Llama 3.1 family of models. Llama 3.2 models are also supported but not recommended for chat, because they are specifically designed to be small or multi-modal.
@@ -71,6 +71,28 @@ Together AI provides fast and reliable inference of open-source models. You'll b
 }
 ```
 
+
+## Novita AI
+
+[Novita AI](https://novita.ai?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) offers an affordable, reliable, and simple inference platform with scalable [LLM API](https://novita.ai/docs/model-api/reference/introduction.html), empowering developers to build AI applications. Try the [Novita AI Llama 3 API Demo](https://novita.ai/model-api/product/llm-api/playground/meta-llama-llama-3.1-70b-instruct?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) today!
+
+1. Create an account [here](https://novita.ai/user/login?&redirect=/&utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link)
+2. Copy your API key that appears on the welcome screen
+3. Update your Continue config file like this:
+
+```json title="config.json"
+{
+  "models": [
+    {
+      "title": "Llama 3.1 405b",
+      "provider": "novita",
+      "model": "meta-llama/llama-3.1-405b-instruct",
+      "apiKey": "<API_KEY>"
+    }
+  ]
+}
+```
+
 ## Replicate
 
 Replicate makes it easy to host and run open-source AI with an API.

docs/docusaurus.config.js

@@ -388,7 +388,11 @@ const config = {
           },
           {
             to: "/customize/model-providers/more/together",
-            from: "/reference/Model Providers/togetherllm",
+            from: "/reference/Model Providers/together",
+          },
+          {
+            to: "/customize/model-providers/more/novita",
+            from: "/reference/Model Providers/novita",
           },
           {
             to: "/customize/model-providers/more/vllm",

docs/i18n/zh-CN/docusaurus-plugin-content-docs/current/chat/model-setup.mdx

@@ -33,7 +33,7 @@ import TabItem from "@theme/TabItem";
 
 ### 来自 Meta 的 Llama 3.1 405B
 
-如果你倾向于使用开放权重模型，那么来自 Meta 的 Llama 3.1 405B 是你当前的最好选择。你需要决定，通过 SaaS 模型提供者使用它（比如 [Together](../customize/model-providers/more/together.md) 或 [Groq](../customize/model-providers/more/groq.md)）或者自托管使用它（比如使用 [vLLM](../customize/model-providers//more/vllm.md) 或 [Ollama](../customize/model-providers/top-level/ollama.md)） 。
+如果你倾向于使用开放权重模型，那么来自 Meta 的 Llama 3.1 405B 是你当前的最好选择。你需要决定，通过 SaaS 模型提供者使用它（比如 [Together](../customize/model-providers/more/together.md) 、[Novita AI](../customize/model-providers/more/novita.md)或 [Groq](../customize/model-providers/more/groq.md)）或者自托管使用它（比如使用 [vLLM](../customize/model-providers//more/vllm.md) 或 [Ollama](../customize/model-providers/top-level/ollama.md)） 。
 
 <Tabs groupId="providers">
     <TabItem value="Together">
@@ -48,6 +48,18 @@ import TabItem from "@theme/TabItem";
             ]
         ```
     </TabItem>
+    <TabItem value="Novita">
+        ```json title="config.json"
+            "models": [
+                {
+                    "title": "Llama 3.1 405B",
+                    "provider": "novita",
+                    "model": "meta-llama/llama-3.1-405b-instruct",
+                    "apiKey": "[NOVITA_API_KEY]"
+                }
+            ]
+        ```
+    </TabItem>
     <TabItem value="Groq">
         ```json title="config.json"
             "models": [

docs/i18n/zh-CN/docusaurus-plugin-content-docs/current/customize/model-providers/more/novita.md

@@ -0,0 +1,18 @@
+# Novita
+
+[Novita AI](https://novita.ai?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) 提供了一个经济实惠、可靠且简单的推理平台。你可以在 [这里](https://novita.ai/user/login?&redirect=/&utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) 注册，在 [Key Management](https://novita.ai/settings/key-management?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link)复制你的 API key ，然后在 [Novita 模型列表](https://novita.ai/llm-api?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) 的任何模型上点击 Try it now 按钮。修改 `~/.continue/config.json` 像这样：
+
+```json title="config.json"
+{
+  "models": [
+    {
+      "title": "Llama 3.1 8b",
+      "provider": "Novita",
+      "model": "meta-llama/llama-3.1-8b-instruct",
+      "apiKey": "YOUR_API_KEY"
+    }
+  ]
+}
+```
+
+[查看代码](https://github.com/continuedev/continue/blob/main/core/llm/llms/Novita.ts)

docs/i18n/zh-CN/docusaurus-plugin-content-docs/current/customize/tutorials/llama3.1.md

@@ -1,7 +1,7 @@
 ---
 title: Continue 使用 Llama 3.1
 description: Continue 如何使用 Llama 3.1
-keywords: [llama, meta, togetherai, ollama, replicate]
+keywords: [llama, meta, togetherai, novita, ollama, replicate]
 ---
 
 Continue 让使用最新的开元模型编码变得简单，包括整个 Llama 3.1 家族模型。
@@ -71,6 +71,27 @@ Together AI 提供开源模型的快速和可信任的推理。你可以以良
 }
 ```
 
+## Novita AI
+
+[Novita AI](https://novita.ai?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) 提供了一个经济实惠、可靠且简单的推理平台。你可以以良好的速度运行 405b 模型。
+
+1. 创建账号 [在这里](https://novita.ai/user/login?&redirect=/&utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link)
+2. 复制[Key Management](https://novita.ai/settings/key-management?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link)中的你的 API key
+3. 更新你的 Continue 配置文件，像这样：
+
+```json title="config.json"
+{
+  "models": [
+    {
+      "title": "Llama 3.1 405b",
+      "provider": "novita",
+      "model": "meta-llama/llama-3.1-405b-instruct",
+      "apiKey": "<API_KEY>"
+    }
+  ]
+}
+```
+
 ## Replicate
 
 Replicate 让使用 API 托管和运行开源 AI 变得简单。

extensions/vscode/config_schema.json

@@ -180,6 +180,7 @@
             "bedrockimport",
             "sagemaker",
             "together",
+            "novita",
             "ollama",
             "huggingface-tgi",
             "huggingface-inference-api",
@@ -222,6 +223,7 @@
             "### Bedrock Imported Models\nTo get started with Bedrock you need to sign up on AWS [here](https://aws.amazon.com/bedrock)",
             "### Sagemaker\nSagemaker is AWS' machine learning platform.",
             "### Together\nTogether is a hosted service that provides extremely fast streaming of open-source language models. To get started with Together:\n1. Obtain an API key from [here](https://together.ai)\n2. Paste below\n3. Select a model preset\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/togetherllm)",
+            "### Novita AI\n[Novita AI](https://novita.ai?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link) offers an affordable, reliable, and simple inference platform with scalable [LLM APIs](https://novita.ai/docs/model-api/reference/introduction.html), empowering developers to build AI applications. To get started with Novita AI:\n1. Obtain an API key from [here](https://novita.ai/settings/key-management?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link)\n2. Paste below\n3. Select a model preset\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/novita)",
             "### Ollama\nTo get started with Ollama, follow these steps:\n1. Download from [ollama.ai](https://ollama.ai/) and open the application\n2. Open a terminal and run `ollama run <MODEL_NAME>`. Example model names are `codellama:7b-instruct` or `llama2:7b-text`. You can find the full list [here](https://ollama.ai/library).\n3. Make sure that the model name used in step 2 is the same as the one in config.json (e.g. `model=\"codellama:7b-instruct\"`)\n4. Once the model has finished downloading, you can start asking questions through Continue.\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/ollama)",
             "### Huggingface TGI\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/huggingfacetgi)",
             "### Huggingface Inference API\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/huggingfaceinferenceapi)",
@@ -262,7 +264,7 @@
         },
         "apiKey": {
           "title": "Api Key",
-          "description": "OpenAI, Anthropic, Cohere, Together, or other API key",
+          "description": "OpenAI, Anthropic, Cohere, Together, Novita AI, or other API key",
           "type": "string"
         },
         "apiBase": {
@@ -475,6 +477,7 @@
                   "huggingface-inference-api",
                   "replicate",
                   "together",
+                  "novita",
                   "cloudflare",
                   "sambanova",
                   "nebius",
@@ -980,6 +983,44 @@
             }
           }
         },
+        {
+          "if": {
+            "properties": {
+              "provider": {
+                "enum": ["novita"]
+              }
+            },
+            "required": ["provider"]
+          },
+          "then": {
+            "properties": {
+              "model": {
+                "anyOf": [
+                  {
+                    "enum": [
+                      "llama3-8b",
+                      "llama3-70b",
+                      "llama3.1-8b",
+                      "llama3.1-70b",
+                      "llama3.1-405b",
+                      "llama3.2-1b",
+                      "llama3.2-3b",
+                      "llama3.2-11b",
+                      "llama3.3-70b",
+                      "mistral-nemo",
+                      "mistral-7b"
+                    ]
+                  },
+                  {
+                    "type": "string"
+                  }
+                ],
+                "markdownDescription": "Select a pre-defined option, or find an exact model string from Novita AI [here](https://novita.ai/llm-api?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link).",
+                "x-intellij-html-description": "Select a pre-defined option, or find an exact model string from Novita AI <a href='https://novita.ai/llm-api?utm_source=github_continuedev&utm_medium=github_readme&utm_campaign=github_link'>here</a>."
+              }
+            }
+          }
+        },
         {
           "if": {
             "properties": {