diff --git a/core/llm/llms/HuggingFaceTGI.ts b/core/llm/llms/HuggingFaceTGI.ts index 5170b8f872..2941589f8d 100644 --- a/core/llm/llms/HuggingFaceTGI.ts +++ b/core/llm/llms/HuggingFaceTGI.ts @@ -8,6 +8,25 @@ class HuggingFaceTGI extends BaseLLM { apiBase: "http://localhost:8080", }; + constructor(options: LLMOptions) { + super(options); + + this.fetch(`${this.apiBase}/info`, { + method: "GET", + }).then(async (response) => { + if (response.status !== 200) { + console.warn( + "Error calling Hugging Face TGI /info endpoint: ", + await response.text() + ); + return; + } + const json = await response.json(); + this.model = json.model_id; + this.contextLength = parseInt(json.max_input_length); + }); + } + private _convertArgs(options: CompletionOptions, prompt: string) { const finalOptions = { max_new_tokens: options.maxTokens,