feat: add MyScale as vectorStore provider (#1)

feat: add MyScale as vectorStore provider
lemonade-hq · Apr 21, 2023 · 613d3b7 · 613d3b7
1 parent f78f2e9
commit 613d3b7
Show file tree

Hide file tree

Showing 12 changed files with 462 additions and 0 deletions.
diff --git a/docs/docs/modules/indexes/vector_stores/integrations/myscale.md b/docs/docs/modules/indexes/vector_stores/integrations/myscale.md
@@ -0,0 +1,76 @@
+---
+sidebar_class_name: node-only
+---
+
+# MyScale
+
+[MyScale](https://myscale.com/) is an emerging AI database that harmonizes the power of vector search and SQL analytics, providing a managed, efficient, and responsive experience.
+
+:::tip Compatibility
+Only available on Node.js.
+:::
+
+## Setup
+
+1. Launch a cluster through [MyScale's Web Console](https://console.myscale.com/), see [the MyScale documentation](https://docs.myscale.com/en/quickstart/) for more information.
+2. Install the Node.js SDK.
+
+```bash npm2yarn
+npm install -S @clickhouse/client
+```
+
+## Index and query docs
+
+```typescript
+import { MyScaleStore } from "langchain/vectorstores/myscale";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+const vectorStore = await MyScaleStore.fromTexts(
+  ["Hello world", "Bye bye", "hello nice world"],
+  [
+    { id: 2, name: "2" },
+    { id: 1, name: "1" },
+    { id: 3, name: "3" },
+  ],
+  new OpenAIEmbeddings(),
+  {
+    host: process.env.MYSCALE_HOST || "https://localhost:8443",
+    username: process.env.MYSCALE_USERNAME || "username",
+    password: process.env.MYSCALE_PASSWORD || "password",
+  }
+);
+
+const results = await vectorStore.similaritySearch("hello world", 1);
+console.log(results);
+
+const filteredResults = await vectorStore.similaritySearch("hello world", 1, {
+  whereStr: "metadata.name = '1'",
+});
+console.log(filteredResults);
+```
+
+## Query docs from existing collection
+
+```typescript
+import { MyScaleStore } from "langchain/vectorstores/myscale";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+const vectorStore = await MyScaleStore.fromExistingIndex(
+  new OpenAIEmbeddings(),
+  {
+    host: process.env.MYSCALE_HOST || "https://localhost:8443",
+    username: process.env.MYSCALE_USERNAME || "username",
+    password: process.env.MYSCALE_PASSWORD || "password",
+    database: "your_database", // default is default
+    table: "your_table", // default is vector_table
+  }
+);
+
+const results = await vectorStore.similaritySearch("hello world", 1);
+console.log(results);
+
+const filteredResults = await vectorStore.similaritySearch("hello world", 1, {
+  whereStr: "metadata.name = '1'",
+});
+console.log(filteredResults);
+```
diff --git a/examples/.env.example b/examples/.env.example
@@ -14,3 +14,6 @@ SUPABASE_URL=ADD_YOURS_HERE # # https://app.supabase.com/project/YOUR_PROJECT_ID
 WEAVIATE_HOST=ADD_YOURS_HERE
 WEAVIATE_SCHEME=ADD_YOURS_HERE
 WEAVIATE_API_KEY=ADD_YOURS_HERE
+MYSCALE_HOST=ADD_YOURS_HERE
+MYSCALE_USERNAME=ADD_YOURS_HERE
+MYSCALE_PASSWORD=ADD_YOURS_HERE
diff --git a/examples/package.json b/examples/package.json
@@ -22,6 +22,7 @@
   "author": "LangChain",
   "license": "MIT",
   "dependencies": {
+    "@clickhouse/client": "^0.0.14",
     "@getmetal/metal-sdk": "^2.0.1",
     "@opensearch-project/opensearch": "^2.2.0",
     "@pinecone-database/pinecone": "^0.0.12",

diff --git a/examples/src/indexes/vector_stores/myscale.ts b/examples/src/indexes/vector_stores/myscale.ts
@@ -0,0 +1,28 @@
+import { MyScaleStore } from "langchain/vectorstores/myscale";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+export async function run() {
+  // Create a store and fill it with some texts + metadata
+  const vectorStore = await MyScaleStore.fromTexts(
+    ["Hello world", "Bye bye", "hello nice world"],
+    [
+      { id: 2, name: "2" },
+      { id: 1, name: "1" },
+      { id: 3, name: "3" },
+    ],
+    new OpenAIEmbeddings(),
+    {
+      host: process.env.MYSCALE_HOST || "https://localhost:8443",
+      username: process.env.MYSCALE_USERNAME || "username",
+      password: process.env.MYSCALE_PASSWORD || "password",
+    }
+  );
+
+  const results = await vectorStore.similaritySearch("hello world", 1);
+  console.log(results);
+
+  const filteredResults = await vectorStore.similaritySearch("hello world", 1, {
+    whereStr: "metadata.name = '1'",
+  });
+  console.log(filteredResults);
+}
diff --git a/langchain/.env.example b/langchain/.env.example
@@ -18,3 +18,6 @@ MILVUS_URL=ADD_YOURS_HERE
 WEAVIATE_HOST=ADD_YOURS_HERE
 WEAVIATE_SCHEME=ADD_YOURS_HERE
 WEAVIATE_API_KEY=ADD_YOURS_HERE
+MYSCALE_HOST=ADD_YOURS_HERE
+MYSCALE_USERNAME=ADD_YOURS_HERE
+MYSCALE_PASSWORD=ADD_YOURS_HERE
diff --git a/langchain/.gitignore b/langchain/.gitignore
@@ -103,6 +103,9 @@ vectorstores/milvus.d.ts
 vectorstores/prisma.cjs
 vectorstores/prisma.js
 vectorstores/prisma.d.ts
+vectorstores/myscale.cjs
+vectorstores/myscale.js
+vectorstores/myscale.d.ts
 text_splitter.cjs
 text_splitter.js
 text_splitter.d.ts

diff --git a/langchain/package.json b/langchain/package.json
@@ -115,6 +115,9 @@
     "vectorstores/prisma.cjs",
     "vectorstores/prisma.js",
     "vectorstores/prisma.d.ts",
+    "vectorstores/myscale.cjs",
+    "vectorstores/myscale.js",
+    "vectorstores/myscale.d.ts",
     "text_splitter.cjs",
     "text_splitter.js",
     "text_splitter.d.ts",
@@ -278,6 +281,7 @@
   "devDependencies": {
     "@aws-sdk/client-lambda": "^3.310.0",
     "@aws-sdk/client-s3": "^3.310.0",
+    "@clickhouse/client": "^0.0.14",
     "@faker-js/faker": "^7.6.0",
     "@getmetal/metal-sdk": "^2.0.1",
     "@huggingface/inference": "^1.5.1",
@@ -333,6 +337,7 @@
   "peerDependencies": {
     "@aws-sdk/client-lambda": "^3.310.0",
     "@aws-sdk/client-s3": "^3.310.0",
+    "@clickhouse/client": "^0.0.14",
     "@getmetal/metal-sdk": "*",
     "@huggingface/inference": "^1.5.1",
     "@opensearch-project/opensearch": "*",
@@ -365,6 +370,9 @@
     "@aws-sdk/client-s3": {
       "optional": true
     },
+    "@clickhouse/client": {
+      "optional": true
+    },
     "@getmetal/metal-sdk": {
       "optional": true
     },
@@ -660,6 +668,11 @@
       "import": "./vectorstores/prisma.js",
       "require": "./vectorstores/prisma.cjs"
     },
+    "./vectorstores/myscale": {
+      "types": "./vectorstores/myscale.d.ts",
+      "import": "./vectorstores/myscale.js",
+      "require": "./vectorstores/myscale.cjs"
+    },
     "./text_splitter": {
       "types": "./text_splitter.d.ts",
       "import": "./text_splitter.js",

diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js
@@ -50,6 +50,7 @@ const entrypoints = {
   "vectorstores/opensearch": "vectorstores/opensearch",
   "vectorstores/milvus": "vectorstores/milvus",
   "vectorstores/prisma": "vectorstores/prisma",
+  "vectorstores/myscale": "vectorstores/myscale",
   // text_splitter
   text_splitter: "text_splitter",
   // memory
@@ -145,6 +146,7 @@ const requiresOptionalDependency = [
   "vectorstores/supabase",
   "vectorstores/opensearch",
   "vectorstores/milvus",
+  "vectorstores/myscale",
   "document_loaders/web/cheerio",
   "document_loaders/web/puppeteer",
   "document_loaders/web/playwright",