From d9c4aaf46e527c933316a41e1074dd84e67b794f Mon Sep 17 00:00:00 2001 From: ryjiang Date: Thu, 26 Sep 2024 19:22:09 +0800 Subject: [PATCH 1/5] fix: queryIterator limit field should be optional (#361) Signed-off-by: ryjiang --- milvus/const/limits.ts | 4 +- milvus/grpc/Data.ts | 5 +++ milvus/types/Data.ts | 2 +- test/grpc/Iterator.spec.ts | 75 +++++++++++++++++++++++++++++++++++++- 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/milvus/const/limits.ts b/milvus/const/limits.ts index 17595287..5a978dd2 100644 --- a/milvus/const/limits.ts +++ b/milvus/const/limits.ts @@ -1,3 +1,5 @@ // limits export const MAX_PARTITION_KEY_FIELD_COUNT = 1; -export const MAX_PARTITIONS_NUMBER = 4096; \ No newline at end of file +export const MAX_PARTITIONS_NUMBER = 4096; + +export const NO_LIMIT = -1; \ No newline at end of file diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index 298a5b62..7a21cc74 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -65,6 +65,7 @@ import { sparseRowsToBytes, getSparseDim, f32ArrayToBinaryBytes, + NO_LIMIT, } from '../'; import { Collection } from './Collection'; @@ -704,6 +705,10 @@ export class Data extends Collection { collection_name: data.collection_name, expr: userExpr, }); + // if limit not set, set it to count + if (!data.limit || data.limit === NO_LIMIT) { + data.limit = count.data; + } // total should be the minimum of total and count const total = data.limit > count.data ? count.data : data.limit; const batchSize = diff --git a/milvus/types/Data.ts b/milvus/types/Data.ts index d2434515..b984c2b2 100644 --- a/milvus/types/Data.ts +++ b/milvus/types/Data.ts @@ -418,7 +418,7 @@ export interface QueryReq extends collectionNameReq { export interface QueryIteratorReq extends Omit { - limit: number; + limit?: number; batchSize: number; } diff --git a/test/grpc/Iterator.spec.ts b/test/grpc/Iterator.spec.ts index b0fd5db4..f70ccd64 100644 --- a/test/grpc/Iterator.spec.ts +++ b/test/grpc/Iterator.spec.ts @@ -1,4 +1,4 @@ -import { MilvusClient, DataType } from '../../milvus'; +import { MilvusClient, DataType, NO_LIMIT } from '../../milvus'; import { IP, genCollectionParams, @@ -299,6 +299,79 @@ describe(`Iterator API`, () => { }); }); + it(`query iterator with limit unset should success`, async () => { + // page size + const batchSize = 5000; + const iterator = await milvusClient.queryIterator({ + collection_name: COLLECTION, + batchSize: batchSize, + expr: 'id > 0', + output_fields: ['id'], + }); + + const results: any = []; + let page = 0; + for await (const value of iterator) { + results.push(...value); + page += 1; + } + + // page size should equal to page + expect(page).toEqual(Math.ceil(data.length / batchSize)); + // results length should equal to data length + expect(results.length).toEqual(data.length); + + // results id should be unique + const idSet = new Set(); + results.forEach((result: any) => { + idSet.add(result.id); + }); + expect(idSet.size).toEqual(data.length); + + // every id in query result should be founded in the original data + results.forEach((result: any) => { + const item = dataMap.get(result.id.toString()); + expect(typeof item !== 'undefined').toEqual(true); + }); + }); + + it(`query iterator with limit = -1 should success`, async () => { + // page size + const batchSize = 5000; + const iterator = await milvusClient.queryIterator({ + collection_name: COLLECTION, + batchSize: batchSize, + expr: 'id > 0', + output_fields: ['id'], + limit: NO_LIMIT, + }); + + const results: any = []; + let page = 0; + for await (const value of iterator) { + results.push(...value); + page += 1; + } + + // page size should equal to page + expect(page).toEqual(Math.ceil(data.length / batchSize)); + // results length should equal to data length + expect(results.length).toEqual(data.length); + + // results id should be unique + const idSet = new Set(); + results.forEach((result: any) => { + idSet.add(result.id); + }); + expect(idSet.size).toEqual(data.length); + + // every id in query result should be founded in the original data + results.forEach((result: any) => { + const item = dataMap.get(result.id.toString()); + expect(typeof item !== 'undefined').toEqual(true); + }); + }); + // it('search iterator with batch size = total should success', async () => { // const batchSize = 100; // const total = 100; From 37f9a51780416f85f3f9773db85ad528f3808d3e Mon Sep 17 00:00:00 2001 From: ryjiang Date: Tue, 1 Oct 2024 10:33:50 +0800 Subject: [PATCH 2/5] update next.js demo (#362) Signed-off-by: shanghaikid --- examples/nextjs/app/page.tsx | 25 ++- examples/nextjs/package.json | 10 +- examples/nextjs/yarn.lock | 294 +++++++++++++++++++++++++++-------- 3 files changed, 253 insertions(+), 76 deletions(-) diff --git a/examples/nextjs/app/page.tsx b/examples/nextjs/app/page.tsx index 63803c5d..c6919751 100644 --- a/examples/nextjs/app/page.tsx +++ b/examples/nextjs/app/page.tsx @@ -1,8 +1,8 @@ import { MilvusClient } from '@zilliz/milvus2-sdk-node'; -async function getData() { +async function getData(address: string) { const milvusClient = new MilvusClient({ - address: 'localhost:19530', + address, }); let res: any = await milvusClient.getMetric({ @@ -17,7 +17,24 @@ async function getData() { } export default async function Home() { - const data = await getData(); + const address = `10.102.5.223:19530`; - return <>{JSON.stringify(data)}; + const data = await getData(address); + + return ( +
+
+

+ Welcome to Milvus +

+

{address}

+ +
+ 
+            {JSON.stringify(data, null, 2)}
+
+
+
+
+ ); } diff --git a/examples/nextjs/package.json b/examples/nextjs/package.json index bab90fca..45a26a58 100644 --- a/examples/nextjs/package.json +++ b/examples/nextjs/package.json @@ -9,10 +9,12 @@ "lint": "next lint" }, "dependencies": { - "@zilliz/milvus2-sdk-node": "^2.4.3", - "next": "^14.2.4", - "react": "^18", - "react-dom": "^18" + "@uiw/react-codemirror": "^4.23.3", + "@zilliz/milvus2-sdk-node": "^2.4.9", + "codemirror": "^6.0.1", + "next": "^14.2.13", + "react": "^18.3.1", + "react-dom": "^18.3.1" }, "devDependencies": { "@types/node": "^20", diff --git a/examples/nextjs/yarn.lock b/examples/nextjs/yarn.lock index 39d96c3d..19173e16 100644 --- a/examples/nextjs/yarn.lock +++ b/examples/nextjs/yarn.lock @@ -7,6 +7,87 @@ resolved "https://registry.yarnpkg.com/@alloc/quick-lru/-/quick-lru-5.2.0.tgz#7bf68b20c0a350f936915fcae06f58e32007ce30" integrity sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw== +"@babel/runtime@^7.18.6": + version "7.25.6" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.25.6.tgz#9afc3289f7184d8d7f98b099884c26317b9264d2" + integrity sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ== + dependencies: + regenerator-runtime "^0.14.0" + +"@codemirror/autocomplete@^6.0.0": + version "6.18.1" + resolved "https://registry.yarnpkg.com/@codemirror/autocomplete/-/autocomplete-6.18.1.tgz#3bd8d62c9c9a14d0706ab0a8adac139eaf1a41f1" + integrity sha512-iWHdj/B1ethnHRTwZj+C1obmmuCzquH29EbcKr0qIjA9NfDeBDJ7vs+WOHsFeLeflE4o+dHfYndJloMKHUkWUA== + dependencies: + "@codemirror/language" "^6.0.0" + "@codemirror/state" "^6.0.0" + "@codemirror/view" "^6.17.0" + "@lezer/common" "^1.0.0" + +"@codemirror/commands@^6.0.0", "@codemirror/commands@^6.1.0": + version "6.6.2" + resolved "https://registry.yarnpkg.com/@codemirror/commands/-/commands-6.6.2.tgz#a8ddb191e00dcc0efa03ea1ff8dc486f902dab91" + integrity sha512-Fq7eWOl1Rcbrfn6jD8FPCj9Auaxdm5nIK5RYOeW7ughnd/rY5AmPg6b+CfsG39ZHdwiwe8lde3q8uR7CF5S0yQ== + dependencies: + "@codemirror/language" "^6.0.0" + "@codemirror/state" "^6.4.0" + "@codemirror/view" "^6.27.0" + "@lezer/common" "^1.1.0" + +"@codemirror/language@^6.0.0": + version "6.10.3" + resolved "https://registry.yarnpkg.com/@codemirror/language/-/language-6.10.3.tgz#eb25fc5ade19032e7bf1dcaa957804e5f1660585" + integrity sha512-kDqEU5sCP55Oabl6E7m5N+vZRoc0iWqgDVhEKifcHzPzjqCegcO4amfrYVL9PmPZpl4G0yjkpTpUO/Ui8CzO8A== + dependencies: + "@codemirror/state" "^6.0.0" + "@codemirror/view" "^6.23.0" + "@lezer/common" "^1.1.0" + "@lezer/highlight" "^1.0.0" + "@lezer/lr" "^1.0.0" + style-mod "^4.0.0" + +"@codemirror/lint@^6.0.0": + version "6.8.2" + resolved "https://registry.yarnpkg.com/@codemirror/lint/-/lint-6.8.2.tgz#7864b03583e9efd18554cff1dd4504da10338ab1" + integrity sha512-PDFG5DjHxSEjOXk9TQYYVjZDqlZTFaDBfhQixHnQOEVDDNHUbEh/hstAjcQJaA6FQdZTD1hquXTK0rVBLADR1g== + dependencies: + "@codemirror/state" "^6.0.0" + "@codemirror/view" "^6.0.0" + crelt "^1.0.5" + +"@codemirror/search@^6.0.0": + version "6.5.6" + resolved "https://registry.yarnpkg.com/@codemirror/search/-/search-6.5.6.tgz#8f858b9e678d675869112e475f082d1e8488db93" + integrity sha512-rpMgcsh7o0GuCDUXKPvww+muLA1pDJaFrpq/CCHtpQJYz8xopu4D1hPcKRoDD0YlF8gZaqTNIRa4VRBWyhyy7Q== + dependencies: + "@codemirror/state" "^6.0.0" + "@codemirror/view" "^6.0.0" + crelt "^1.0.5" + +"@codemirror/state@^6.0.0", "@codemirror/state@^6.1.1", "@codemirror/state@^6.4.0": + version "6.4.1" + resolved "https://registry.yarnpkg.com/@codemirror/state/-/state-6.4.1.tgz#da57143695c056d9a3c38705ed34136e2b68171b" + integrity sha512-QkEyUiLhsJoZkbumGZlswmAhA7CBU02Wrz7zvH4SrcifbsqwlXShVXg65f3v/ts57W3dqyamEriMhij1Z3Zz4A== + +"@codemirror/theme-one-dark@^6.0.0": + version "6.1.2" + resolved "https://registry.yarnpkg.com/@codemirror/theme-one-dark/-/theme-one-dark-6.1.2.tgz#fcef9f9cfc17a07836cb7da17c9f6d7231064df8" + integrity sha512-F+sH0X16j/qFLMAfbciKTxVOwkdAS336b7AXTKOZhy8BR3eH/RelsnLgLFINrpST63mmN2OuwUt0W2ndUgYwUA== + dependencies: + "@codemirror/language" "^6.0.0" + "@codemirror/state" "^6.0.0" + "@codemirror/view" "^6.0.0" + "@lezer/highlight" "^1.0.0" + +"@codemirror/view@^6.0.0", "@codemirror/view@^6.17.0", "@codemirror/view@^6.23.0", "@codemirror/view@^6.27.0": + version "6.34.1" + resolved "https://registry.yarnpkg.com/@codemirror/view/-/view-6.34.1.tgz#b17ed29c563e4adc60086233f2d3e7197e2dc33e" + integrity sha512-t1zK/l9UiRqwUNPm+pdIT0qzJlzuVckbTEMVNFhfWkGiBQClstzg+78vedCvLSX0xJEZ6lwZbPpnljL7L6iwMQ== + dependencies: + "@codemirror/state" "^6.4.0" + style-mod "^4.1.0" + w3c-keyname "^2.2.4" + "@colors/colors@1.6.0", "@colors/colors@^1.6.0": version "1.6.0" resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.6.0.tgz#ec6cd237440700bc23ca23087f513c75508958b0" @@ -139,10 +220,29 @@ resolved "https://registry.yarnpkg.com/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz#9299f82874bab9e4c7f9c48d865becbfe8d6907c" integrity sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw== -"@next/env@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.4.tgz#5546813dc4f809884a37d257b254a5ce1b0248d7" - integrity sha512-3EtkY5VDkuV2+lNmKlbkibIJxcO4oIHEhBWne6PaAp+76J9KoSsGvNikp6ivzAT8dhhBMYrm6op2pS1ApG0Hzg== +"@lezer/common@^1.0.0", "@lezer/common@^1.1.0": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@lezer/common/-/common-1.2.1.tgz#198b278b7869668e1bebbe687586e12a42731049" + integrity sha512-yemX0ZD2xS/73llMZIK6KplkjIjf2EvAHcinDi/TfJ9hS25G0388+ClHt6/3but0oOxinTcQHJLDXh6w1crzFQ== + +"@lezer/highlight@^1.0.0": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@lezer/highlight/-/highlight-1.2.1.tgz#596fa8f9aeb58a608be0a563e960c373cbf23f8b" + integrity sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA== + dependencies: + "@lezer/common" "^1.0.0" + +"@lezer/lr@^1.0.0": + version "1.4.2" + resolved "https://registry.yarnpkg.com/@lezer/lr/-/lr-1.4.2.tgz#931ea3dea8e9de84e90781001dae30dea9ff1727" + integrity sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA== + dependencies: + "@lezer/common" "^1.0.0" + +"@next/env@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.13.tgz#ba341ba9eb70db428fc1c754f49c3c516f7bab47" + integrity sha512-s3lh6K8cbW1h5Nga7NNeXrbe0+2jIIYK9YaA9T7IufDWnZpozdFUp6Hf0d5rNWUKu4fEuSX2rCKlGjCrtylfDw== "@next/eslint-plugin-next@13.5.4": version "13.5.4" @@ -151,50 +251,50 @@ dependencies: glob "7.1.7" -"@next/swc-darwin-arm64@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.4.tgz#da9f04c34a3d5f0b8401ed745768420e4a604036" - integrity sha512-AH3mO4JlFUqsYcwFUHb1wAKlebHU/Hv2u2kb1pAuRanDZ7pD/A/KPD98RHZmwsJpdHQwfEc/06mgpSzwrJYnNg== - -"@next/swc-darwin-x64@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.4.tgz#46dedb29ec5503bf171a72a3ecb8aac6e738e9d6" - integrity sha512-QVadW73sWIO6E2VroyUjuAxhWLZWEpiFqHdZdoQ/AMpN9YWGuHV8t2rChr0ahy+irKX5mlDU7OY68k3n4tAZTg== - -"@next/swc-linux-arm64-gnu@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.4.tgz#c9697ab9eb422bd1d7ffd0eb0779cc2aefa9d4a1" - integrity sha512-KT6GUrb3oyCfcfJ+WliXuJnD6pCpZiosx2X3k66HLR+DMoilRb76LpWPGb4tZprawTtcnyrv75ElD6VncVamUQ== - -"@next/swc-linux-arm64-musl@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.4.tgz#cbbceb2008571c743b5a310a488d2e166d200a75" - integrity sha512-Alv8/XGSs/ytwQcbCHwze1HmiIkIVhDHYLjczSVrf0Wi2MvKn/blt7+S6FJitj3yTlMwMxII1gIJ9WepI4aZ/A== - -"@next/swc-linux-x64-gnu@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.4.tgz#d79184223f857bacffb92f643cb2943a43632568" - integrity sha512-ze0ShQDBPCqxLImzw4sCdfnB3lRmN3qGMB2GWDRlq5Wqy4G36pxtNOo2usu/Nm9+V2Rh/QQnrRc2l94kYFXO6Q== - -"@next/swc-linux-x64-musl@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.4.tgz#6b6c3e5ac02ca5e63394d280ec8ee607491902df" - integrity sha512-8dwC0UJoc6fC7PX70csdaznVMNr16hQrTDAMPvLPloazlcaWfdPogq+UpZX6Drqb1OBlwowz8iG7WR0Tzk/diQ== - -"@next/swc-win32-arm64-msvc@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.4.tgz#dbad3906e870dba84c5883d9d4c4838472e0697f" - integrity sha512-jxyg67NbEWkDyvM+O8UDbPAyYRZqGLQDTPwvrBBeOSyVWW/jFQkQKQ70JDqDSYg1ZDdl+E3nkbFbq8xM8E9x8A== - -"@next/swc-win32-ia32-msvc@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.4.tgz#6074529b91ba49132922ce89a2e16d25d2ec235d" - integrity sha512-twrmN753hjXRdcrZmZttb/m5xaCBFa48Dt3FbeEItpJArxriYDunWxJn+QFXdJ3hPkm4u7CKxncVvnmgQMY1ag== - -"@next/swc-win32-x64-msvc@14.2.4": - version "14.2.4" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.4.tgz#e65a1c6539a671f97bb86d5183d6e3a1733c29c7" - integrity sha512-tkLrjBzqFTP8DVrAAQmZelEahfR9OxWpFR++vAI9FBhCiIxtwHwBHC23SBHCTURBtwB4kc/x44imVOnkKGNVGg== +"@next/swc-darwin-arm64@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.13.tgz#76f08d78360c4d27d444df7f35a56f59a48f4808" + integrity sha512-IkAmQEa2Htq+wHACBxOsslt+jMoV3msvxCn0WFSfJSkv/scy+i/EukBKNad36grRxywaXUYJc9mxEGkeIs8Bzg== + +"@next/swc-darwin-x64@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.13.tgz#1d4821d54bb01dacc6a6c32408f8468a4f4af269" + integrity sha512-Dv1RBGs2TTjkwEnFMVL5XIfJEavnLqqwYSD6LXgTPdEy/u6FlSrLBSSfe1pcfqhFEXRAgVL3Wpjibe5wXJzWog== + +"@next/swc-linux-arm64-gnu@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.13.tgz#79d9af8d3408df9990c8911889eca1ca6a308f19" + integrity sha512-yB1tYEFFqo4ZNWkwrJultbsw7NPAAxlPXURXioRl9SdW6aIefOLS+0TEsKrWBtbJ9moTDgU3HRILL6QBQnMevg== + +"@next/swc-linux-arm64-musl@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.13.tgz#b13180645865b120591db2f1e831743ebc02ab36" + integrity sha512-v5jZ/FV/eHGoWhMKYrsAweQ7CWb8xsWGM/8m1mwwZQ/sutJjoFaXchwK4pX8NqwImILEvQmZWyb8pPTcP7htWg== + +"@next/swc-linux-x64-gnu@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.13.tgz#8cb8480dfeee512648e4e08c2095aac0461b876f" + integrity sha512-aVc7m4YL7ViiRv7SOXK3RplXzOEe/qQzRA5R2vpXboHABs3w8vtFslGTz+5tKiQzWUmTmBNVW0UQdhkKRORmGA== + +"@next/swc-linux-x64-musl@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.13.tgz#df5ca922fa1e1ee81b15a06a2d3d3ace0efd2bd7" + integrity sha512-4wWY7/OsSaJOOKvMsu1Teylku7vKyTuocvDLTZQq0TYv9OjiYYWt63PiE1nTuZnqQ4RPvME7Xai+9enoiN0Wrg== + +"@next/swc-win32-arm64-msvc@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.13.tgz#8a7db6e71f526212587975f743b28e4d1cb829d1" + integrity sha512-uP1XkqCqV2NVH9+g2sC7qIw+w2tRbcMiXFEbMihkQ8B1+V6m28sshBwAB0SDmOe0u44ne1vFU66+gx/28RsBVQ== + +"@next/swc-win32-ia32-msvc@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.13.tgz#6aa664f36f2d70c5ae6ffcbbc56784d33f24522d" + integrity sha512-V26ezyjPqQpDBV4lcWIh8B/QICQ4v+M5Bo9ykLN+sqeKKBxJVDpEc6biDVyluTXTC40f5IqCU0ttth7Es2ZuMw== + +"@next/swc-win32-x64-msvc@14.2.13": + version "14.2.13" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.13.tgz#5a920eea82a58affa6146192586716cec6c87fed" + integrity sha512-WwzOEAFBGhlDHE5Z73mNU8CO8mqMNLqaG+AO9ETmzdCQlJhVtWZnOl2+rqgVQS+YHunjOWptdFmNfbpwcUuEsw== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -381,15 +481,40 @@ "@typescript-eslint/types" "6.21.0" eslint-visitor-keys "^3.4.1" +"@uiw/codemirror-extensions-basic-setup@4.23.3": + version "4.23.3" + resolved "https://registry.yarnpkg.com/@uiw/codemirror-extensions-basic-setup/-/codemirror-extensions-basic-setup-4.23.3.tgz#f2dec646a6a1a6072b8b73e67372cb9cd178537e" + integrity sha512-nEMjgbCyeLx+UQgOGAAoUWYFE34z5TlyaKNszuig/BddYFDb0WKcgmC37bDFxR2dZssf3K/lwGWLpXnGKXePbA== + dependencies: + "@codemirror/autocomplete" "^6.0.0" + "@codemirror/commands" "^6.0.0" + "@codemirror/language" "^6.0.0" + "@codemirror/lint" "^6.0.0" + "@codemirror/search" "^6.0.0" + "@codemirror/state" "^6.0.0" + "@codemirror/view" "^6.0.0" + +"@uiw/react-codemirror@^4.23.3": + version "4.23.3" + resolved "https://registry.yarnpkg.com/@uiw/react-codemirror/-/react-codemirror-4.23.3.tgz#2febc96478e206d704a8810306b9beab1ee76af1" + integrity sha512-TBBLUbeqXmfQSfO+f3rPNOAb+QXbSm7KPB64FHQWLGg2WJNbpOhjLOWMyL+C4ZP3aSCNc2Y5aftEK1vp3wCKTA== + dependencies: + "@babel/runtime" "^7.18.6" + "@codemirror/commands" "^6.1.0" + "@codemirror/state" "^6.1.1" + "@codemirror/theme-one-dark" "^6.0.0" + "@uiw/codemirror-extensions-basic-setup" "4.23.3" + codemirror "^6.0.0" + "@ungap/structured-clone@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@zilliz/milvus2-sdk-node@^2.4.3": - version "2.4.3" - resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-2.4.3.tgz#7e5fbef63a7450884d55e1b42ad69d0f8ebd30eb" - integrity sha512-BLXFe6V/AkFJtWotZV33w5MeT5ofbE3ZeSa3ddB3mY3Ug0AiBMbEw1DqKGepRndANVveHKVdkP3uLhb4iaxpFA== +"@zilliz/milvus2-sdk-node@^2.4.9": + version "2.4.9" + resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-2.4.9.tgz#760f78b44a720426aad775746254d23eb4e7dfc8" + integrity sha512-EuCwEE5jENkxk7JVCNxLjtw4BgxcucquzIOalqxhOBQ10P28wU+jFU1ETJpAen4g5FbNwlMCO9rwDaORKjlwSA== dependencies: "@grpc/grpc-js" "^1.8.22" "@grpc/proto-loader" "^0.7.10" @@ -729,6 +854,19 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" +codemirror@^6.0.0, codemirror@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-6.0.1.tgz#62b91142d45904547ee3e0e0e4c1a79158035a29" + integrity sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg== + dependencies: + "@codemirror/autocomplete" "^6.0.0" + "@codemirror/commands" "^6.0.0" + "@codemirror/language" "^6.0.0" + "@codemirror/lint" "^6.0.0" + "@codemirror/search" "^6.0.0" + "@codemirror/state" "^6.0.0" + "@codemirror/view" "^6.0.0" + color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -787,6 +925,11 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== +crelt@^1.0.5: + version "1.0.6" + resolved "https://registry.yarnpkg.com/crelt/-/crelt-1.0.6.tgz#7cc898ea74e190fb6ef9dae57f8f81cf7302df72" + integrity sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g== + cross-spawn@^7.0.0, cross-spawn@^7.0.2: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -2108,12 +2251,12 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== -next@^14.2.4: - version "14.2.4" - resolved "https://registry.yarnpkg.com/next/-/next-14.2.4.tgz#ef66c39c71e2d8ad0a3caa0383c8933f4663e4d1" - integrity sha512-R8/V7vugY+822rsQGQCjoLhMuC9oFj9SOi4Cl4b2wjDrseD0LRZ10W7R6Czo4w9ZznVSshKjuIomsRjvm9EKJQ== +next@^14.2.13: + version "14.2.13" + resolved "https://registry.yarnpkg.com/next/-/next-14.2.13.tgz#32da2ee0afbe729e2d4a467c3570def90e1c974d" + integrity sha512-BseY9YNw8QJSwLYD7hlZzl6QVDoSFHL/URN5K64kVEVpCsSOWeyjbIGK+dZUaRViHTaMQX8aqmnn0PHBbGZezg== dependencies: - "@next/env" "14.2.4" + "@next/env" "14.2.13" "@swc/helpers" "0.5.5" busboy "1.6.0" caniuse-lite "^1.0.30001579" @@ -2121,15 +2264,15 @@ next@^14.2.4: postcss "8.4.31" styled-jsx "5.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "14.2.4" - "@next/swc-darwin-x64" "14.2.4" - "@next/swc-linux-arm64-gnu" "14.2.4" - "@next/swc-linux-arm64-musl" "14.2.4" - "@next/swc-linux-x64-gnu" "14.2.4" - "@next/swc-linux-x64-musl" "14.2.4" - "@next/swc-win32-arm64-msvc" "14.2.4" - "@next/swc-win32-ia32-msvc" "14.2.4" - "@next/swc-win32-x64-msvc" "14.2.4" + "@next/swc-darwin-arm64" "14.2.13" + "@next/swc-darwin-x64" "14.2.13" + "@next/swc-linux-arm64-gnu" "14.2.13" + "@next/swc-linux-arm64-musl" "14.2.13" + "@next/swc-linux-x64-gnu" "14.2.13" + "@next/swc-linux-x64-musl" "14.2.13" + "@next/swc-win32-arm64-msvc" "14.2.13" + "@next/swc-win32-ia32-msvc" "14.2.13" + "@next/swc-win32-x64-msvc" "14.2.13" node-releases@^2.0.14: version "2.0.14" @@ -2444,7 +2587,7 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== -react-dom@^18: +react-dom@^18.3.1: version "18.3.1" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.3.1.tgz#c2265d79511b57d479b3dd3fdfa51536494c5cb4" integrity sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw== @@ -2457,7 +2600,7 @@ react-is@^16.13.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== -react@^18: +react@^18.3.1: version "18.3.1" resolved "https://registry.yarnpkg.com/react/-/react-18.3.1.tgz#49ab892009c53933625bd16b2533fc754cab2891" integrity sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ== @@ -2500,6 +2643,11 @@ reflect.getprototypeof@^1.0.4: globalthis "^1.0.3" which-builtin-type "^1.1.3" +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== + regexp.prototype.flags@^1.5.1, regexp.prototype.flags@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334" @@ -2810,6 +2958,11 @@ strip-json-comments@^3.1.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== +style-mod@^4.0.0, style-mod@^4.1.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/style-mod/-/style-mod-4.1.2.tgz#ca238a1ad4786520f7515a8539d5a63691d7bf67" + integrity sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw== + styled-jsx@5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/styled-jsx/-/styled-jsx-5.1.1.tgz#839a1c3aaacc4e735fed0781b8619ea5d0009d1f" @@ -3032,6 +3185,11 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== +w3c-keyname@^2.2.4: + version "2.2.8" + resolved "https://registry.yarnpkg.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz#7b17c8c6883d4e8b86ac8aba79d39e880f8869c5" + integrity sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ== + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" From 500ad2b7aaeb7fc4c67cd60e201e42551c5f5858 Mon Sep 17 00:00:00 2001 From: shanghaikid Date: Tue, 1 Oct 2024 10:35:08 +0800 Subject: [PATCH 3/5] update milvus demo sdk version Signed-off-by: shanghaikid --- examples/milvus/package.json | 6 +- examples/milvus/yarn.lock | 227 ++++++++++++++++++++++------------- 2 files changed, 145 insertions(+), 88 deletions(-) diff --git a/examples/milvus/package.json b/examples/milvus/package.json index e83e625d..9e6ba965 100644 --- a/examples/milvus/package.json +++ b/examples/milvus/package.json @@ -8,11 +8,11 @@ "benchp": "autocannon http://localhost:4000 -c 1 -p 10" }, "dependencies": { - "@zilliz/milvus2-sdk-node": "^2.4.3", - "express": "^4.19.2" + "@zilliz/milvus2-sdk-node": "^2.4.9", + "express": "^4.21.0" }, "devDependencies": { "@types/express": "^4.17.17", - "autocannon": "^7.11.0" + "autocannon": "^7.15.0" } } diff --git a/examples/milvus/yarn.lock b/examples/milvus/yarn.lock index e67fab47..974449e8 100644 --- a/examples/milvus/yarn.lock +++ b/examples/milvus/yarn.lock @@ -194,10 +194,10 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.2.tgz#38ecb64f01aa0d02b7c8f4222d7c38af6316fef8" integrity sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g== -"@zilliz/milvus2-sdk-node@^2.4.3": - version "2.4.3" - resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-2.4.3.tgz#7e5fbef63a7450884d55e1b42ad69d0f8ebd30eb" - integrity sha512-BLXFe6V/AkFJtWotZV33w5MeT5ofbE3ZeSa3ddB3mY3Ug0AiBMbEw1DqKGepRndANVveHKVdkP3uLhb4iaxpFA== +"@zilliz/milvus2-sdk-node@^2.4.9": + version "2.4.9" + resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-2.4.9.tgz#760f78b44a720426aad775746254d23eb4e7dfc8" + integrity sha512-EuCwEE5jENkxk7JVCNxLjtw4BgxcucquzIOalqxhOBQ10P28wU+jFU1ETJpAen4g5FbNwlMCO9rwDaORKjlwSA== dependencies: "@grpc/grpc-js" "^1.8.22" "@grpc/proto-loader" "^0.7.10" @@ -243,10 +243,10 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== -autocannon@^7.11.0: - version "7.11.0" - resolved "https://registry.yarnpkg.com/autocannon/-/autocannon-7.11.0.tgz#4f52dd15a90707da76de6a7122cc74cd30271981" - integrity sha512-17farET7ImAZjOCGSxiG3TcEbc6b+zWWQ61CpKC8D6UIjq+U3c1foRDSMk8Rgl4VJ01Hbbbx6UhdXhye9yTEZg== +autocannon@^7.15.0: + version "7.15.0" + resolved "https://registry.yarnpkg.com/autocannon/-/autocannon-7.15.0.tgz#e1bd5b9bfb4f3983864b988f565b5f5b4e74b2e4" + integrity sha512-NaP2rQyA+tcubOJMFv2+oeW9jv2pq/t+LM6BL3cfJic0HEfscEcnWgAyU5YovE/oTHUzAgTliGdLPR+RQAWUbg== dependencies: chalk "^4.1.0" char-spinner "^1.0.1" @@ -277,10 +277,10 @@ base64-js@^1.2.0: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -body-parser@1.20.2: - version "1.20.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" - integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== +body-parser@1.20.3: + version "1.20.3" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6" + integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g== dependencies: bytes "3.1.2" content-type "~1.0.5" @@ -290,7 +290,7 @@ body-parser@1.20.2: http-errors "2.0.0" iconv-lite "0.4.24" on-finished "2.4.1" - qs "6.11.0" + qs "6.13.0" raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" @@ -300,13 +300,16 @@ bytes@3.1.2: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== -call-bind@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" - integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== +call-bind@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9" + integrity sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w== dependencies: - function-bind "^1.1.1" - get-intrinsic "^1.0.2" + es-define-property "^1.0.0" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" + set-function-length "^1.2.1" chalk@^4.1.0: version "4.1.2" @@ -438,6 +441,15 @@ debug@2.6.9: dependencies: ms "2.0.0" +define-data-property@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" + integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== + dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" + gopd "^1.0.1" + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -473,6 +485,23 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== +encodeurl@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" + integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== + +es-define-property@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845" + integrity sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ== + dependencies: + get-intrinsic "^1.2.4" + +es-errors@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" + integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== + escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -488,37 +517,37 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -express@^4.19.2: - version "4.19.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" - integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== +express@^4.21.0: + version "4.21.0" + resolved "https://registry.yarnpkg.com/express/-/express-4.21.0.tgz#d57cb706d49623d4ac27833f1cbc466b668eb915" + integrity sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.2" + body-parser "1.20.3" content-disposition "0.5.4" content-type "~1.0.4" cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "1.2.0" + finalhandler "1.3.1" fresh "0.5.2" http-errors "2.0.0" - merge-descriptors "1.0.1" + merge-descriptors "1.0.3" methods "~1.1.2" on-finished "2.4.1" parseurl "~1.3.3" - path-to-regexp "0.1.7" + path-to-regexp "0.1.10" proxy-addr "~2.0.7" - qs "6.11.0" + qs "6.13.0" range-parser "~1.2.1" safe-buffer "5.2.1" - send "0.18.0" - serve-static "1.15.0" + send "0.19.0" + serve-static "1.16.2" setprototypeof "1.2.0" statuses "2.0.1" type-is "~1.6.18" @@ -530,13 +559,13 @@ fecha@^4.2.0: resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw== -finalhandler@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" - integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== +finalhandler@1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.3.1.tgz#0c575f1d1d324ddd1da35ad7ece3df7d19088019" + integrity sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ== dependencies: debug "2.6.9" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" on-finished "2.4.1" parseurl "~1.3.3" @@ -567,10 +596,10 @@ fresh@0.5.2: resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== -function-bind@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== generic-pool@^3.9.0: version "3.9.0" @@ -582,15 +611,23 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.2: - version "1.2.1" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.1.tgz#d295644fed4505fc9cde952c37ee12b477a83d82" - integrity sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw== +get-intrinsic@^1.1.3, get-intrinsic@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" + integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== dependencies: - function-bind "^1.1.1" - has "^1.0.3" + es-errors "^1.3.0" + function-bind "^1.1.2" has-proto "^1.0.1" has-symbols "^1.0.3" + hasown "^2.0.0" + +gopd@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" + integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA== + dependencies: + get-intrinsic "^1.1.3" has-async-hooks@^1.0.0: version "1.0.0" @@ -602,6 +639,13 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== +has-property-descriptors@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" + integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== + dependencies: + es-define-property "^1.0.0" + has-proto@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.1.tgz#1885c1305538958aff469fef37937c22795408e0" @@ -612,12 +656,12 @@ has-symbols@^1.0.3: resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== -has@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" - integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== +hasown@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== dependencies: - function-bind "^1.1.1" + function-bind "^1.1.2" hdr-histogram-js@^3.0.0: version "3.0.0" @@ -753,10 +797,10 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== -merge-descriptors@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" - integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w== +merge-descriptors@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz#d80319a65f3c7935351e5cfdac8f9318504dbed5" + integrity sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ== methods@~1.1.2: version "1.1.2" @@ -800,10 +844,10 @@ negotiator@0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== -object-inspect@^1.9.0: - version "1.12.3" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.3.tgz#ba62dffd67ee256c8c086dfae69e016cd1f198b9" - integrity sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g== +object-inspect@^1.13.1: + version "1.13.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.2.tgz#dea0088467fb991e67af4058147a24824a3043ff" + integrity sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g== on-finished@2.4.1: version "2.4.1" @@ -834,10 +878,10 @@ parseurl@~1.3.3: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== -path-to-regexp@0.1.7: - version "0.1.7" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" - integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== +path-to-regexp@0.1.10: + version "0.1.10" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.10.tgz#67e9108c5c0551b9e5326064387de4763c4d5f8b" + integrity sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w== pretty-bytes@^5.4.1: version "5.6.0" @@ -875,12 +919,12 @@ proxy-addr@~2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" -qs@6.11.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" - integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== +qs@6.13.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" + integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg== dependencies: - side-channel "^1.0.4" + side-channel "^1.0.6" range-parser@~1.2.1: version "1.2.1" @@ -943,10 +987,10 @@ semver@^7.3.2: dependencies: lru-cache "^6.0.0" -send@0.18.0: - version "0.18.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" - integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== +send@0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8" + integrity sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw== dependencies: debug "2.6.9" depd "2.0.0" @@ -962,29 +1006,42 @@ send@0.18.0: range-parser "~1.2.1" statuses "2.0.1" -serve-static@1.15.0: - version "1.15.0" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" - integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== +serve-static@1.16.2: + version "1.16.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.2.tgz#b6a5343da47f6bdd2673848bf45754941e803296" + integrity sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw== dependencies: - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.18.0" + send "0.19.0" + +set-function-length@^1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" + integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== + dependencies: + define-data-property "^1.1.4" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" + gopd "^1.0.1" + has-property-descriptors "^1.0.2" setprototypeof@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== -side-channel@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" - integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== +side-channel@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" + integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== dependencies: - call-bind "^1.0.0" - get-intrinsic "^1.0.2" - object-inspect "^1.9.0" + call-bind "^1.0.7" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" + object-inspect "^1.13.1" simple-swizzle@^0.2.2: version "0.2.2" From 88c2ee009fbc5ff41d6a9b6a069758647ea80321 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Tue, 8 Oct 2024 14:52:18 +0800 Subject: [PATCH 4/5] support clustering key (#363) * support clustering key Signed-off-by: ryjiang * update test case Signed-off-by: ryjiang --------- Signed-off-by: ryjiang --- milvus/types/Collection.ts | 2 + milvus/utils/Format.ts | 3 + test/grpc/Basic.spec.ts | 10 ++- test/grpc/ClusteringKey.spec.ts | 111 ++++++++++++++++++++++++++++++++ test/tools/collection.ts | 6 ++ test/utils/Format.spec.ts | 5 +- 6 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 test/grpc/ClusteringKey.spec.ts diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts index eeba9927..6493d129 100644 --- a/milvus/types/Collection.ts +++ b/milvus/types/Collection.ts @@ -67,6 +67,7 @@ export interface FieldType { element_type?: DataType | keyof typeof DataTypeMap; is_primary_key?: boolean; is_partition_key?: boolean; + is_clustering_key?: boolean; type_params?: { [key: string]: TypeParam; }; @@ -98,6 +99,7 @@ export interface BaseCreateCollectionReq extends GrpcTimeOut { | 'Customized'; // optional,consistency level, default is 'Bounded' num_partitions?: number; // optional, partitions number, default is 1 partition_key_field?: string; // optional, partition key field + clustring_key_field?: string; // optional, clustring key field enable_dynamic_field?: boolean; // optional, enable dynamic field, default is false enableDynamicField?: boolean; // optional, alias of enable_dynamic_field properties?: Properties; diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index b5e8aa47..b4fb3247 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -295,6 +295,7 @@ export const formatCollectionSchema = ( enable_dynamic_field, enableDynamicField, partition_key_field, + clustring_key_field, } = data; let fields = (data as CreateCollectionWithFieldsReq).fields; @@ -318,6 +319,8 @@ export const formatCollectionSchema = ( isPrimaryKey: !!field.is_primary_key, isPartitionKey: !!field.is_partition_key || field.name === partition_key_field, + isClusteringKey: + !!field.is_clustering_key || field.name === clustring_key_field, }; // if element type exist and diff --git a/test/grpc/Basic.spec.ts b/test/grpc/Basic.spec.ts index 49762eb9..e88f55f2 100644 --- a/test/grpc/Basic.spec.ts +++ b/test/grpc/Basic.spec.ts @@ -1,7 +1,7 @@ import { MilvusClient, ErrorCode, DataType } from '../../milvus'; import { IP, GENERATE_NAME, generateInsertData } from '../tools'; -const milvusClient = new MilvusClient({ address: IP }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION_NAME = GENERATE_NAME(); const schema = [ { @@ -49,6 +49,14 @@ describe(`Basic API without database`, () => { collection_name: COLLECTION_NAME, }); expect(desc.schema.fields.length).toEqual(schema.length); + expect(desc.schema.fields[0].name).toEqual('vector'); + expect(desc.schema.fields[1].name).toEqual('id'); + expect(desc.schema.fields[2].name).toEqual('varChar'); + expect(desc.schema.fields[3].name).toEqual('array'); + // test primary key + expect(desc.schema.fields[1].is_primary_key).toEqual(true); + // test partition key + expect(desc.schema.fields[2].is_partition_key).toEqual(false); }); it(`Create index should be successful`, async () => { diff --git a/test/grpc/ClusteringKey.spec.ts b/test/grpc/ClusteringKey.spec.ts new file mode 100644 index 00000000..cb473b1e --- /dev/null +++ b/test/grpc/ClusteringKey.spec.ts @@ -0,0 +1,111 @@ +import { MilvusClient, DataType, ErrorCode } from '../../milvus'; +import { IP, GENERATE_NAME, genCollectionParams } from '../tools'; + +const milvusClient = new MilvusClient({ address: IP }); +const COLLECTION_NAME = GENERATE_NAME(); +const COLLECTION_NAME2 = GENERATE_NAME(); +const dbParam = { + db_name: 'ClusteringKey', +}; + +describe(`Clustering key API`, () => { + beforeAll(async () => { + await milvusClient.createDatabase(dbParam); + await milvusClient.use(dbParam); + }); + + afterAll(async () => { + await milvusClient.dropCollection({ + collection_name: COLLECTION_NAME, + }); + await milvusClient.dropCollection({ + collection_name: COLLECTION_NAME2, + }); + await milvusClient.dropDatabase(dbParam); + }); + + it(`Create Collection with schema should successfully`, async () => { + const res = await milvusClient.createCollection( + genCollectionParams({ + collectionName: COLLECTION_NAME, + dim: [4], + vectorType: [DataType.FloatVector], + autoID: true, + clusterKeyEnabled: true, + }) + ); + expect(res.error_code).toEqual(ErrorCode.SUCCESS); + + // describe + const desc = await milvusClient.describeCollection({ + collection_name: COLLECTION_NAME, + }); + + // test clustering key + let found = 0; + desc.schema.fields.forEach(field => { + if (field.is_clustering_key) { + found++; + } + }); + + expect(found).toEqual(1); + }); + + it(`Create Collection should be successful with clusteringkey`, async () => { + const schema = [ + { + name: 'vector', + description: 'Vector field', + data_type: DataType.FloatVector, + dim: Number(4), + }, + { + name: 'id', + description: 'ID field', + data_type: DataType.Int64, + is_primary_key: true, + autoID: true, + }, + { + name: 'varChar', + description: 'VarChar field', + data_type: DataType.VarChar, + max_length: 128, + is_partition_key: false, + is_clustering_key: false, + }, + { + name: 'array', + description: 'array field', + data_type: DataType.Array, + element_type: DataType.VarChar, + max_capacity: 128, + max_length: 128, + is_partition_key: false, + }, + ]; + const res = await milvusClient.createCollection({ + collection_name: COLLECTION_NAME2, + fields: schema, + clustring_key_field: 'varChar', + }); + expect(res.error_code).toEqual(ErrorCode.SUCCESS); + + // describe + const desc = await milvusClient.describeCollection({ + collection_name: COLLECTION_NAME2, + }); + + // test clustering key + let found = 0; + desc.schema.fields.forEach(field => { + if (field.is_clustering_key) { + found++; + expect(field.name).toEqual('varChar'); + } + }); + + expect(found).toEqual(1); + }); +}); diff --git a/test/tools/collection.ts b/test/tools/collection.ts index 7740c71b..e1f83851 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -37,6 +37,7 @@ export const genCollectionParams = (data: { autoID?: boolean; fields?: any[]; partitionKeyEnabled?: boolean; + clusterKeyEnabled?: boolean; numPartitions?: number; enableDynamic?: boolean; maxCapacity?: number; @@ -53,6 +54,7 @@ export const genCollectionParams = (data: { enableDynamic = false, maxCapacity, idType = DataType.Int64, + clusterKeyEnabled = false, } = data; const vectorFields = vectorType.map((type, i) => { @@ -152,5 +154,9 @@ export const genCollectionParams = (data: { params.num_partitions = numPartitions; } + if (clusterKeyEnabled) { + params.clustring_key_field = 'int64'; + } + return params; }; diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index e98d34d0..8317f2b7 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -301,6 +301,7 @@ describe('utils/format', () => { dataType: 5, isPrimaryKey: true, isPartitionKey: false, + isClusteringKey: false, }, { typeParams: [ @@ -317,6 +318,7 @@ describe('utils/format', () => { dataType: 101, isPrimaryKey: false, isPartitionKey: false, + isClusteringKey: false, }, { typeParams: [ @@ -334,6 +336,7 @@ describe('utils/format', () => { isPrimaryKey: false, isPartitionKey: false, elementType: 5, + isClusteringKey: false, }, ], }; @@ -866,7 +869,7 @@ describe('utils/format', () => { describeCollectionResponse, milvusProto ); - console.dir(searchRequest, { depth: null }); + // console.dir(searchRequest, { depth: null }); expect(searchRequest.isHybridSearch).toEqual(true); expect(searchRequest.request.collection_name).toEqual('test'); expect(searchRequest.request.output_fields).toEqual(['vector', 'vector1']); From ab7357d93ae78567c48967ca959d5b330794b543 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Thu, 24 Oct 2024 11:09:54 +0800 Subject: [PATCH 5/5] [2.5] Support Functions (#365) * init 2.5 Signed-off-by: ryjiang * update milvus version Signed-off-by: ryjiang * update proto Signed-off-by: ryjiang * [2.5] support functions (#364) * WIP: functions Signed-off-by: ryjiang * WIP: functions Signed-off-by: ryjiang * WIP Signed-off-by: ryjiang * WIP Signed-off-by: ryjiang * suppport functions part2 Signed-off-by: ryjiang * functions part3 Signed-off-by: ryjiang * finish functions Signed-off-by: ryjiang * update test Signed-off-by: ryjiang --------- Signed-off-by: ryjiang --------- Signed-off-by: ryjiang --- .github/workflows/check.yml | 4 +- .gitmodules | 2 +- milvus/const/milvus.ts | 7 + milvus/grpc/BaseClient.ts | 14 +- milvus/grpc/Collection.ts | 19 ++- milvus/grpc/Data.ts | 2 + milvus/types/Collection.ts | 33 +++- milvus/types/Data.ts | 17 ++- milvus/utils/Bytes.ts | 61 ++++---- milvus/utils/Format.ts | 132 +++++++++++----- milvus/utils/Grpc.ts | 4 +- package.json | 2 +- proto | 2 +- test/grpc/Functions.spec.ts | 242 ++++++++++++++++++++++++++++++ test/grpc/MultipleVectors.spec.ts | 2 +- test/tools/collection.ts | 15 +- test/tools/data.ts | 5 + test/utils/Format.spec.ts | 145 +++++++++++------- 18 files changed, 548 insertions(+), 160 deletions(-) create mode 100644 test/grpc/Functions.spec.ts diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8376c8f4..d90e01c6 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -4,12 +4,12 @@ on: pull_request: branches: - main - - 2.4 + - 2.5 types: [opened, synchronize] push: branches: - main - - 2.4 + - 2.5 jobs: publish: runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index f1271435..b5e19c85 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "proto"] path = proto url = https://github.com/milvus-io/milvus-proto.git - branch = 2.4 + branch = master diff --git a/milvus/const/milvus.ts b/milvus/const/milvus.ts index a76bdabf..977101ef 100644 --- a/milvus/const/milvus.ts +++ b/milvus/const/milvus.ts @@ -117,6 +117,8 @@ export enum MetricType { SUBSTRUCTURE = 'SUBSTRUCTURE', // SUPERSTRUCTURE superstructure SUPERSTRUCTURE = 'SUPERSTRUCTURE', + // BM 25 + BM25 = 'BM25', } // Index types @@ -279,6 +281,11 @@ export enum DataType { SparseFloatVector = 104, } +export enum FunctionType { + Unknown = 0, + BM25 = 1, +} + export const VectorDataTypes = [ DataType.BinaryVector, DataType.FloatVector, diff --git a/milvus/grpc/BaseClient.ts b/milvus/grpc/BaseClient.ts index 383f83a7..a3294dc8 100644 --- a/milvus/grpc/BaseClient.ts +++ b/milvus/grpc/BaseClient.ts @@ -67,15 +67,13 @@ export class BaseClient { protected schemaProto: Root; // The Milvus protobuf. protected milvusProto: Root; - // The milvus collection schema Type - protected collectionSchemaType: Type; - // The milvus field schema Type - protected fieldSchemaType: Type; + // milvus proto protected readonly protoInternalPath = { serviceName: 'milvus.proto.milvus.MilvusService', collectionSchema: 'milvus.proto.schema.CollectionSchema', fieldSchema: 'milvus.proto.schema.FieldSchema', + functionSchema: 'milvus.proto.schema.FunctionSchema', }; /** @@ -136,14 +134,6 @@ export class BaseClient { this.schemaProto = protobuf.loadSync(this.protoFilePath.schema); this.milvusProto = protobuf.loadSync(this.protoFilePath.milvus); - // Get the CollectionSchemaType and FieldSchemaType from the schemaProto object. - this.collectionSchemaType = this.schemaProto.lookupType( - this.protoInternalPath.collectionSchema - ); - this.fieldSchemaType = this.schemaProto.lookupType( - this.protoInternalPath.fieldSchema - ); - // options this.channelOptions = { // Milvus default max_receive_message_length is 100MB, but Milvus support change max_receive_message_length . diff --git a/milvus/grpc/Collection.ts b/milvus/grpc/Collection.ts index a1bf553d..b76d1700 100644 --- a/milvus/grpc/Collection.ts +++ b/milvus/grpc/Collection.ts @@ -145,15 +145,28 @@ export class Collection extends Database { validatePartitionNumbers(num_partitions); } + // Get the CollectionSchemaType and FieldSchemaType from the schemaProto object. + const schemaTypes = { + collectionSchemaType: this.schemaProto.lookupType( + this.protoInternalPath.collectionSchema + ), + fieldSchemaType: this.schemaProto.lookupType( + this.protoInternalPath.fieldSchema + ), + functionSchemaType: this.schemaProto.lookupType( + this.protoInternalPath.functionSchema + ), + }; + // Create the payload object with the collection_name, description, and fields. // it should follow CollectionSchema in schema.proto - const payload = formatCollectionSchema(data, this.fieldSchemaType); + const payload = formatCollectionSchema(data, schemaTypes); // Create the collectionParams object from the payload. - const collectionSchema = this.collectionSchemaType.create(payload); + const collectionSchema = schemaTypes.collectionSchemaType.create(payload); // Encode the collectionParams object to bytes. - const schemaBytes = this.collectionSchemaType + const schemaBytes = schemaTypes.collectionSchemaType .encode(collectionSchema) .finish(); diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index 7a21cc74..c53ba45d 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -142,9 +142,11 @@ export class Data extends Collection { // Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`. // If primarykey is set `autoid = true`, you cannot insert the data. + // and if function field is set, you need to ignore the field value in the data. const fieldMap = new Map( collectionInfo.schema.fields .filter(v => !v.is_primary_key || !v.autoID) + .filter(v => !v.is_function_output) .map(v => [ v.name, { diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts index 6493d129..f73ebd1c 100644 --- a/milvus/types/Collection.ts +++ b/milvus/types/Collection.ts @@ -13,6 +13,7 @@ import { LoadState, DataTypeMap, ShowCollectionsType, + FunctionType, } from '../'; // returned from milvus @@ -21,7 +22,7 @@ export interface FieldSchema { index_params: KeyValuePair[]; fieldID: string | number; name: string; - is_primary_key?: boolean; + is_primary_key: boolean; description: string; data_type: keyof typeof DataType; autoID: boolean; @@ -29,9 +30,11 @@ export interface FieldSchema { element_type?: keyof typeof DataType; default_value?: number | string; dataType: DataType; - is_partition_key?: boolean; - is_dynamic?: boolean; - is_clustering_key?: boolean; + is_partition_key: boolean; + is_dynamic: boolean; + is_clustering_key: boolean; + is_function_output: boolean; + nullable: boolean; } export interface CollectionData { @@ -56,7 +59,7 @@ export interface ReplicaInfo { node_ids: string[]; } -export type TypeParam = string | number; +export type TypeParam = string | number | Record; export type TypeParamKey = 'dim' | 'max_length' | 'max_capacity'; // create collection @@ -67,6 +70,7 @@ export interface FieldType { element_type?: DataType | keyof typeof DataTypeMap; is_primary_key?: boolean; is_partition_key?: boolean; + is_function_output?: boolean; is_clustering_key?: boolean; type_params?: { [key: string]: TypeParam; @@ -76,6 +80,9 @@ export interface FieldType { max_capacity?: TypeParam; max_length?: TypeParam; default_value?: number | string; + enable_match?: boolean; + tokenizer_params?: Record; + enable_tokenizer?: boolean; } export interface ShowCollectionsReq extends GrpcTimeOut { @@ -86,6 +93,15 @@ export interface ShowCollectionsReq extends GrpcTimeOut { export type Properties = Record; +export type Function = { + name: string; + description?: string; + type: FunctionType; + input_field_names: string[]; + output_field_names?: string[]; + params: Record; +}; + export interface BaseCreateCollectionReq extends GrpcTimeOut { // collection name collection_name: string; // required, collection name @@ -102,8 +118,9 @@ export interface BaseCreateCollectionReq extends GrpcTimeOut { clustring_key_field?: string; // optional, clustring key field enable_dynamic_field?: boolean; // optional, enable dynamic field, default is false enableDynamicField?: boolean; // optional, alias of enable_dynamic_field - properties?: Properties; - db_name?: string; + properties?: Properties; // optional, collection properties + db_name?: string; // optional, db name + functions?: Function[]; // optionals, doc-in/doc-out functions } export interface CreateCollectionWithFieldsReq extends BaseCreateCollectionReq { @@ -187,6 +204,7 @@ export interface CollectionSchema { enable_dynamic_field: boolean; autoID: boolean; fields: FieldSchema[]; + functions: Function[]; } export interface DescribeCollectionResponse extends TimeStamp { @@ -205,6 +223,7 @@ export interface DescribeCollectionResponse extends TimeStamp { shards_num: number; num_partitions?: string; // int64 db_name: string; + functions: Function[]; } export interface GetCompactionPlansResponse extends resStatusResponse { diff --git a/milvus/types/Data.ts b/milvus/types/Data.ts index b984c2b2..b16b6b66 100644 --- a/milvus/types/Data.ts +++ b/milvus/types/Data.ts @@ -282,7 +282,7 @@ export interface SearchParam { group_by_field?: string; // group by field } -// old search api parameter type +// old search api parameter type, deprecated export interface SearchReq extends collectionNameReq { anns_field?: string; // your vector field name partition_names?: string[]; // partition names @@ -307,13 +307,18 @@ export interface SearchIteratorReq limit: number; } +export type SearchTextType = string | string[]; +export type SearchVectorType = VectorTypes | VectorTypes[]; +export type SearchDataType = SearchVectorType | SearchTextType; +export type SearchMultipleDataType = VectorTypes[] | SearchTextType[]; + // simplified search api parameter type export interface SearchSimpleReq extends collectionNameReq { partition_names?: string[]; // partition names - anns_field?: string; // your vector field name - data?: VectorTypes[] | VectorTypes; // vector to search - vector?: VectorTypes; // alias for data - vectors?: VectorTypes[]; // alias for data + anns_field?: string; // your vector field name，rquired if you are searching on multiple vector fields collection + data?: SearchDataType; // vector or text to search + vector?: VectorTypes; // alias for data, deprecated + vectors?: VectorTypes[]; // alias for data, deprecated output_fields?: string[]; limit?: number; // how many results you want topk?: number; // limit alias @@ -333,7 +338,7 @@ export type HybridSearchSingleReq = Pick< SearchParam, 'anns_field' | 'ignore_growing' | 'group_by_field' > & { - data: VectorTypes[] | VectorTypes; // vector to search + data: SearchDataType; // vector to search expr?: string; // filter expression params?: keyValueObj; // extra search parameters transformers?: OutputTransformers; // provide custom data transformer for specific data type like bf16 or f16 vectors diff --git a/milvus/utils/Bytes.ts b/milvus/utils/Bytes.ts index 66f354b6..3e63f349 100644 --- a/milvus/utils/Bytes.ts +++ b/milvus/utils/Bytes.ts @@ -5,12 +5,13 @@ import { BinaryVector, SparseFloatVector, DataType, - VectorTypes, + SearchMultipleDataType, Float16Vector, SparseVectorCSR, SparseVectorCOO, BFloat16Vector, SparseVectorArray, + FieldSchema, } from '..'; /** @@ -250,41 +251,49 @@ export const bytesToSparseRow = (bufferData: Buffer): SparseFloatVector => { * This function builds a placeholder group in bytes format for Milvus. * * @param {Root} milvusProto - The root object of the Milvus protocol. - * @param {VectorTypes[]} vectors - An array of search vectors. + * @param {SearchMultipleDataType[]} data - An array of search vectors. * @param {DataType} vectorDataType - The data type of the vectors. * * @returns {Uint8Array} The placeholder group in bytes format. */ export const buildPlaceholderGroupBytes = ( milvusProto: Root, - vectors: VectorTypes[], - vectorDataType: DataType + data: SearchMultipleDataType, + field: FieldSchema ) => { + const { dataType, is_function_output } = field; // create placeholder_group value let bytes; - // parse vectors to bytes - switch (vectorDataType) { - case DataType.FloatVector: - bytes = vectors.map(v => f32ArrayToF32Bytes(v as FloatVector)); - break; - case DataType.BinaryVector: - bytes = vectors.map(v => f32ArrayToBinaryBytes(v as BinaryVector)); - break; - case DataType.BFloat16Vector: - bytes = vectors.map(v => - Array.isArray(v) ? f32ArrayToBf16Bytes(v as BFloat16Vector) : v - ); - break; - case DataType.Float16Vector: - bytes = vectors.map(v => - Array.isArray(v) ? f32ArrayToF16Bytes(v as Float16Vector) : v - ); - break; - case DataType.SparseFloatVector: - bytes = vectors.map(v => sparseToBytes(v as SparseFloatVector)); - break; + if (is_function_output) { + // parse text to bytes + bytes = data.map(d => new TextEncoder().encode(String(d))); + } else { + // parse vectors to bytes + switch (dataType) { + case DataType.FloatVector: + bytes = data.map(v => f32ArrayToF32Bytes(v as FloatVector)); + break; + case DataType.BinaryVector: + bytes = data.map(v => f32ArrayToBinaryBytes(v as BinaryVector)); + break; + case DataType.BFloat16Vector: + bytes = data.map(v => + Array.isArray(v) ? f32ArrayToBf16Bytes(v as BFloat16Vector) : v + ); + break; + case DataType.Float16Vector: + bytes = data.map(v => + Array.isArray(v) ? f32ArrayToF16Bytes(v as Float16Vector) : v + ); + break; + case DataType.SparseFloatVector: + bytes = data.map(v => sparseToBytes(v as SparseFloatVector)); + + break; + } } + // create placeholder_group const PlaceholderGroup = milvusProto.lookupType( 'milvus.proto.common.PlaceholderGroup' @@ -295,7 +304,7 @@ export const buildPlaceholderGroupBytes = ( placeholders: [ { tag: '$0', - type: vectorDataType, + type: is_function_output ? DataType.VarChar : dataType, values: bytes, }, ], diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index b4fb3247..44b7c14d 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -41,6 +41,10 @@ import { f32ArrayToF16Bytes, bf16BytesToF32Array, f16BytesToF32Array, + TypeParam, + SearchDataType, + FieldSchema, + SearchMultipleDataType, } from '../'; /** @@ -64,12 +68,18 @@ export const formatKeyValueData = (data: KeyValuePair[], keys: string[]) => { * @param data Object * @returns {KeyValuePair[]} */ -export const parseToKeyValue = (data?: { - [x: string]: any; -}): KeyValuePair[] => { +export const parseToKeyValue = ( + data?: { + [x: string]: any; + }, + valueToString?: boolean +): KeyValuePair[] => { return data ? Object.keys(data).reduce( - (pre: any[], cur: string) => [...pre, { key: cur, value: data[cur] }], + (pre: any[], cur: string) => [ + ...pre, + { key: cur, value: valueToString ? String(data[cur]) : data[cur] }, + ], [] ) : []; @@ -188,21 +198,34 @@ export const formatAddress = (address: string) => { */ export const assignTypeParams = ( field: FieldType, - typeParamKeys: string[] = ['dim', 'max_length', 'max_capacity'] + typeParamKeys: string[] = [ + 'dim', + 'max_length', + 'max_capacity', + 'enable_match', + 'enable_tokenizer', + 'tokenizer_params', + ] ) => { let newField = cloneObj(field); typeParamKeys.forEach(key => { if (newField.hasOwnProperty(key)) { // if the property exists in the field object, assign it to the type_params object newField.type_params = newField.type_params || {}; - newField.type_params[key] = String(newField[key as keyof FieldType]); + newField.type_params[key] = + typeof newField[key as keyof FieldType] !== 'object' + ? String(newField[key as keyof FieldType] ?? '') + : (newField[key as keyof FieldType] as TypeParam); // delete the property from the field object delete newField[key as keyof FieldType]; } if (newField.type_params && newField.type_params[key]) { - // if the property already exists in the type_params object, convert it to a string - newField.type_params[key] = String(newField.type_params[key]); + // if the property already exists in the type_params object, convert it to a string, + newField.type_params[key] = + typeof newField.type_params[key] !== 'object' + ? String(newField.type_params[key]) + : newField.type_params[key]; } }); return newField; @@ -266,7 +289,7 @@ export const convertToDataType = ( throw new Error(ERROR_REASONS.FIELD_TYPE_IS_NOT_SUPPORT); }; -/** +/**dd * Creates a deep copy of the provided object using JSON.parse and JSON.stringify. * Note that this function is not efficient and may cause performance issues if used with large or complex objects. It also does not handle cases where the object being cloned contains functions or prototype methods. * @@ -287,7 +310,7 @@ export const cloneObj = (obj: T): T => { */ export const formatCollectionSchema = ( data: CreateCollectionReq, - fieldSchemaType: Type + schemaTypes: Record ): { [k: string]: any } => { const { collection_name, @@ -295,6 +318,7 @@ export const formatCollectionSchema = ( enable_dynamic_field, enableDynamicField, partition_key_field, + functions, clustring_key_field, } = data; @@ -310,25 +334,33 @@ export const formatCollectionSchema = ( enableDynamicField: !!enableDynamicField || !!enable_dynamic_field, fields: fields.map(field => { // Assign the typeParams property to the result of parseToKeyValue(type_params). - const { type_params, ...rest } = assignTypeParams(field); + const { + type_params, + data_type, + element_type, + is_function_output, + is_partition_key, + is_primary_key, + ...rest + } = assignTypeParams(field); const dataType = convertToDataType(field.data_type); const createObj: any = { ...rest, typeParams: parseToKeyValue(type_params), + data_type, // compatibility with old version dataType, - isPrimaryKey: !!field.is_primary_key, + isPrimaryKey: !!is_primary_key, isPartitionKey: - !!field.is_partition_key || field.name === partition_key_field, + !!is_partition_key || field.name === partition_key_field, + isFunctionOutput: !!is_function_output, isClusteringKey: !!field.is_clustering_key || field.name === clustring_key_field, }; // if element type exist and - if ( - dataType === DataType.Array && - typeof field.element_type !== 'undefined' - ) { - createObj.elementType = convertToDataType(field.element_type); + if (dataType === DataType.Array && typeof element_type !== 'undefined') { + createObj.elementType = convertToDataType(element_type); + createObj.element_type = element_type; // compatibility with old version } if (typeof field.default_value !== 'undefined') { @@ -338,9 +370,23 @@ export const formatCollectionSchema = ( [dataKey]: field.default_value, }; } - return fieldSchemaType.create(createObj); + return schemaTypes.fieldSchemaType.create(createObj); }), - }; + functions: [], + } as any; + + // if functions is set, parse its params to key-value pairs, and delete inputs and outputs + if (functions) { + payload.functions = functions.map((func: any) => { + const { input_field_names, output_field_names, ...rest } = func; + return schemaTypes.functionSchemaType.create({ + ...rest, + inputFieldNames: input_field_names, + outputFieldNames: output_field_names, + params: parseToKeyValue(func.params, true), + }); + }); + } return payload; }; @@ -718,7 +764,9 @@ export const buildSearchRequest = ( // merge single request with hybrid request req = Object.assign(cloneObj(data), singleReq); } else { - // if it is not hybrid search, and we have built one request, skip + // if it is not hybrid search, and we have built one request + // or user has specified an anns_field to search and is not matching + // skip const skip = requests.length === 1 || (typeof req.anns_field !== 'undefined' && req.anns_field !== name); @@ -727,29 +775,29 @@ export const buildSearchRequest = ( } } - // get search vectors - let searchingVector: VectorTypes | VectorTypes[] = isHybridSearch + // get search data + let searchData: SearchDataType | SearchMultipleDataType = isHybridSearch ? req.data! : searchReq.vectors || searchSimpleReq.vectors || searchSimpleReq.vector || searchSimpleReq.data; - // format searching vector - searchingVector = formatSearchVector(searchingVector, field.dataType!); + // format searching data + searchData = formatSearchData(searchData, field); // create search request requests.push({ collection_name: req.collection_name, partition_names: req.partition_names || [], output_fields: req.output_fields || default_output_fields, - nq: searchReq.nq || searchingVector.length, + nq: searchReq.nq || searchData.length, dsl: searchReq.expr || searchSimpleReq.filter || '', dsl_type: DslType.BoolExprV1, placeholder_group: buildPlaceholderGroupBytes( milvusProto, - searchingVector as VectorTypes[], - field.dataType! + searchData as VectorTypes[], + field ), search_params: parseToKeyValue( searchReq.search_params || buildSearchParams(req, name) @@ -885,28 +933,36 @@ export const formatSearchResult = ( /** * Formats the search vector to match a specific data type. - * @param {VectorTypes | VectorTypes[]} searchVector - The search vector or array of vectors to be formatted. + * @param {SearchDataType[]} searchVector - The search vector or array of vectors to be formatted. * @param {DataType} dataType - The specified data type. * @returns {VectorTypes[]} The formatted search vector or array of vectors. */ -export const formatSearchVector = ( - searchVector: VectorTypes | VectorTypes[], - dataType: DataType -): VectorTypes[] => { +export const formatSearchData = ( + searchData: SearchDataType | SearchMultipleDataType, + field: FieldSchema +): SearchMultipleDataType => { + const { dataType, is_function_output } = field; + + if (is_function_output) { + return ( + Array.isArray(searchData) ? searchData : [searchData] + ) as SearchMultipleDataType; + } + switch (dataType) { case DataType.FloatVector: case DataType.BinaryVector: case DataType.Float16Vector: case DataType.BFloat16Vector: - if (!Array.isArray(searchVector)) { - return [searchVector] as VectorTypes[]; + if (!Array.isArray(searchData)) { + return [searchData] as VectorTypes[]; } case DataType.SparseFloatVector: - const type = getSparseFloatVectorType(searchVector as SparseVectorArray); + const type = getSparseFloatVectorType(searchData as SparseVectorArray); if (type !== 'unknown') { - return [searchVector] as VectorTypes[]; + return [searchData] as VectorTypes[]; } default: - return searchVector as VectorTypes[]; + return searchData as VectorTypes[]; } }; diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts index 28459b89..7a450c92 100644 --- a/milvus/utils/Grpc.ts +++ b/milvus/utils/Grpc.ts @@ -200,7 +200,7 @@ export const getRetryInterceptor = ({ logger.debug( `\x1b[32m[Response(${ Date.now() - startTime.getTime() - }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${msg}` + }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${string}` ); savedMessageNext(savedReceiveMessage); @@ -217,7 +217,7 @@ export const getRetryInterceptor = ({ const msg = string.length > 2048 ? string.slice(0, 2048) + '...' : string; logger.debug( - `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}` + `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${string}` ); savedSendMessage = message; next(message); diff --git a/package.json b/package.json index b5f58dba..cc7931a2 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "@zilliz/milvus2-sdk-node", "author": "ued@zilliz.com", + "milvusVersion": "master-20241024-f78f6112-amd64", "version": "2.4.9", - "milvusVersion": "v2.4.11", "main": "dist/milvus", "files": [ "dist" diff --git a/proto b/proto index 32b0311b..85ccff4d 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 32b0311b378567df2a3d93dde7b5cc64fd8b7349 +Subproject commit 85ccff4d57fe9c510c88ee4eaf1ba33ef4ef1188 diff --git a/test/grpc/Functions.spec.ts b/test/grpc/Functions.spec.ts new file mode 100644 index 00000000..f68c1301 --- /dev/null +++ b/test/grpc/Functions.spec.ts @@ -0,0 +1,242 @@ +import { + MilvusClient, + DataType, + ErrorCode, + MetricType, + ConsistencyLevelEnum, + FunctionType, +} from '../../milvus'; +import { + IP, + genCollectionParams, + GENERATE_NAME, + generateInsertData, + dynamicFields, +} from '../tools'; + +const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); +const COLLECTION = GENERATE_NAME(); +const dbParam = { + db_name: 'Functions', +}; +const numPartitions = 3; + +// create +const createCollectionParams = genCollectionParams({ + collectionName: COLLECTION, + dim: [4], + vectorType: [DataType.FloatVector], + autoID: false, + partitionKeyEnabled: true, + numPartitions, + enableDynamic: true, + fields: [ + { + name: 'text', + description: 'text field', + data_type: DataType.VarChar, + max_length: 20, + is_partition_key: false, + enable_tokenizer: true, + }, + { + name: 'sparse', + description: 'sparse field', + data_type: DataType.SparseFloatVector, + is_function_output: true, + }, + { + name: 'sparse2', + description: 'sparse field2', + data_type: DataType.SparseFloatVector, + is_function_output: true, + }, + ], + functions: [ + { + name: 'bm25f1', + description: 'bm25 function', + type: FunctionType.BM25, + input_field_names: ['text'], + output_field_names: ['sparse'], + params: {}, + }, + { + name: 'bm25f2', + description: 'bm25 function', + type: FunctionType.BM25, + input_field_names: ['text'], + output_field_names: ['sparse2'], + params: {}, + }, + ], +}); + +describe(`Functions schema API`, () => { + beforeAll(async () => { + // create db and use db + await milvusClient.createDatabase(dbParam); + await milvusClient.use(dbParam); + }); + afterAll(async () => { + await milvusClient.dropCollection({ + collection_name: COLLECTION, + }); + await milvusClient.dropDatabase(dbParam); + }); + + it(`Create schema with function collection should success`, async () => { + const create = await milvusClient.createCollection(createCollectionParams); + + expect(create.error_code).toEqual(ErrorCode.SUCCESS); + + // describe + const describe = await milvusClient.describeCollection({ + collection_name: COLLECTION, + }); + // expect the 'sparse' field to be created + expect(describe.schema.fields.length).toEqual( + createCollectionParams.fields.length + ); + // extract the 'sparse' field + const sparse = describe.schema.fields.find( + field => field.is_function_output + ); + // expect the 'sparse' field's name to be 'sparse' + expect(sparse!.name).toEqual('sparse'); + + // expect functions are in the schema + expect(describe.schema.functions.length).toEqual(2); + expect(describe.schema.functions[0].name).toEqual('bm25f1'); + expect(describe.schema.functions[0].input_field_names).toEqual(['text']); + expect(describe.schema.functions[0].output_field_names).toEqual(['sparse']); + expect(describe.schema.functions[0].type).toEqual('BM25'); + expect(describe.schema.functions[1].name).toEqual('bm25f2'); + expect(describe.schema.functions[1].input_field_names).toEqual(['text']); + expect(describe.schema.functions[1].output_field_names).toEqual([ + 'sparse2', + ]); + expect(describe.schema.functions[1].type).toEqual('BM25'); + }); + + it(`Insert data with function field should success`, async () => { + const data = generateInsertData( + [...createCollectionParams.fields, ...dynamicFields], + 10 + ); + + const insert = await milvusClient.insert({ + collection_name: COLLECTION, + fields_data: data, + }); + + expect(insert.status.error_code).toEqual(ErrorCode.SUCCESS); + }); + + it(`Create index on function output field should success`, async () => { + // create index + const createVectorIndex = await milvusClient.createIndex({ + collection_name: COLLECTION, + index_name: 't', + field_name: 'vector', + index_type: 'HNSW', + metric_type: MetricType.COSINE, + params: { M: 4, efConstruction: 8 }, + }); + + const createIndex = await milvusClient.createIndex({ + collection_name: COLLECTION, + index_name: 't2', + field_name: 'sparse', + index_type: 'SPARSE_INVERTED_INDEX', + metric_type: 'BM25', + params: { drop_ratio_build: 0.3, bm25_k1: 1.25, bm25_b: 0.8 }, + }); + + const createIndex2 = await milvusClient.createIndex({ + collection_name: COLLECTION, + index_name: 't3', + field_name: 'sparse2', + index_type: 'SPARSE_INVERTED_INDEX', + metric_type: 'BM25', + params: { drop_ratio_build: 0.3, bm25_k1: 1.25, bm25_b: 0.8 }, + }); + + expect(createVectorIndex.error_code).toEqual(ErrorCode.SUCCESS); + expect(createIndex.error_code).toEqual(ErrorCode.SUCCESS); + expect(createIndex2.error_code).toEqual(ErrorCode.SUCCESS); + + // load + const load = await milvusClient.loadCollection({ + collection_name: COLLECTION, + }); + + expect(load.error_code).toEqual(ErrorCode.SUCCESS); + }); + + it(`query with function output field should success`, async () => { + // query + const query = await milvusClient.query({ + collection_name: COLLECTION, + limit: 10, + expr: 'id > 0', + output_fields: ['vector', 'id', 'text', 'sparse', 'sparse2'], + consistency_level: ConsistencyLevelEnum.Strong, + }); + + expect(query.status.error_code).toEqual(ErrorCode.SUCCESS); + expect(query.data.length).toEqual(10); + // data should have 'sparse' field + expect(query.data[0].hasOwnProperty('sparse')).toBeTruthy(); + // data should have 'sparse2' field + expect(query.data[0].hasOwnProperty('sparse2')).toBeTruthy(); + }); + + it(`search with varchar should success`, async () => { + // search nq = 1 + const search = await milvusClient.search({ + collection_name: COLLECTION, + limit: 10, + data: 'apple', + anns_field: 'sparse', + output_fields: ['*'], + params: { drop_ratio_search: 0.6 }, + consistency_level: ConsistencyLevelEnum.Strong, + }); + + expect(search.status.error_code).toEqual(ErrorCode.SUCCESS); + + // nq > 1 + const search2 = await milvusClient.search({ + collection_name: COLLECTION, + limit: 10, + data: ['apple', 'banana'], + anns_field: 'sparse', + output_fields: ['*'], + params: { drop_ratio_search: 0.6 }, + consistency_level: ConsistencyLevelEnum.Strong, + }); + + expect(search2.status.error_code).toEqual(ErrorCode.SUCCESS); + + // multiple search + const search3 = await milvusClient.search({ + collection_name: COLLECTION, + limit: 10, + data: [ + { + data: 'apple', + anns_field: 'sparse', + params: { nprobe: 2 }, + }, + { + data: [1, 2, 3, 4], + anns_field: 'vector', + }, + ], + consistency_level: ConsistencyLevelEnum.Strong, + }); + + expect(search3.status.error_code).toEqual(ErrorCode.SUCCESS); + }); +}); diff --git a/test/grpc/MultipleVectors.spec.ts b/test/grpc/MultipleVectors.spec.ts index f8f24378..d31665c2 100644 --- a/test/grpc/MultipleVectors.spec.ts +++ b/test/grpc/MultipleVectors.spec.ts @@ -16,7 +16,7 @@ import { generateInsertData, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); const COLLECTION_NAME = GENERATE_NAME(); const dbParam = { diff --git a/test/tools/collection.ts b/test/tools/collection.ts index e1f83851..83902ffb 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -1,5 +1,10 @@ -import { DataType, ConsistencyLevelEnum } from '../../milvus'; -import { VECTOR_FIELD_NAME, MAX_CAPACITY, MAX_LENGTH } from './const'; +import { + DataType, + ConsistencyLevelEnum, + FunctionType, + Function, +} from '../../milvus'; +import { MAX_CAPACITY, MAX_LENGTH } from './const'; import { GENERATE_VECTOR_NAME } from './'; export const dynamicFields = [ @@ -42,6 +47,7 @@ export const genCollectionParams = (data: { enableDynamic?: boolean; maxCapacity?: number; idType?: DataType; + functions?: Function[]; }) => { const { collectionName, @@ -54,6 +60,7 @@ export const genCollectionParams = (data: { enableDynamic = false, maxCapacity, idType = DataType.Int64, + functions, clusterKeyEnabled = false, } = data; @@ -117,6 +124,7 @@ export const genCollectionParams = (data: { data_type: DataType.VarChar, max_length: MAX_LENGTH, is_partition_key: partitionKeyEnabled, + enable_tokenizer: true, }, { name: 'json', @@ -154,6 +162,9 @@ export const genCollectionParams = (data: { params.num_partitions = numPartitions; } + if (functions && functions?.length > 0) { + params.functions = functions; + } if (clusterKeyEnabled) { params.clustring_key_field = 'int64'; } diff --git a/test/tools/data.ts b/test/tools/data.ts index f1c4dfb2..b0c8e07b 100644 --- a/test/tools/data.ts +++ b/test/tools/data.ts @@ -295,6 +295,11 @@ export const generateInsertData = ( continue; } + // skip fields with is_function_output = true + if (field.is_function_output) { + continue; + } + // Parameters used to generate all types of data const genDataParams = { dim: Number(field.dim || (field.type_params && field.type_params.dim)), diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index 8317f2b7..f31c57ca 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -26,8 +26,9 @@ import { buildFieldData, formatSearchResult, Field, - formatSearchVector, + formatSearchData, buildSearchRequest, + FieldSchema, } from '../../milvus'; describe('utils/format', () => { @@ -75,6 +76,15 @@ describe('utils/format', () => { ]); }); + it(`should convert {row_count:4, b: 3} t0 [{key:"row_count",value:'4'}, {key: "b", value: '4'}]`, () => { + const testValue = { row_count: '4', b: 3 }; + const res = parseToKeyValue(testValue, true); + expect(res).toMatchObject([ + { key: 'row_count', value: '4' }, + { key: 'b', value: '3' }, + ]); + }); + it(`should convert [{key:"row_count",value:4}] to {row_count:4}`, () => { const testValue = 3.1231241241234124124; const res = formatNumberPrecision(testValue, 3); @@ -166,12 +176,15 @@ describe('utils/format', () => { expect(methodName).toBe('123'); }); - it('should assign properties with keys `dim` or `max_length` to the `type_params` object and delete them from the `field` object', () => { + it('should assign properties with keys `dim` or `max_length` to the `type_params`, `enable_match`, `tokenizer_params`, `enable_tokenizer` object and delete them from the `field` object', () => { const field = { name: 'vector', data_type: 'BinaryVector', dim: 128, max_length: 100, + enable_match: true, + tokenizer_params: { key: 'value' }, + enable_tokenizer: true, } as FieldType; const expectedOutput = { name: 'vector', @@ -179,6 +192,9 @@ describe('utils/format', () => { type_params: { dim: '128', max_length: '100', + enable_match: 'true', + tokenizer_params: { key: 'value' }, + enable_tokenizer: 'true', }, }; expect(assignTypeParams(field)).toEqual(expectedOutput); @@ -261,7 +277,7 @@ describe('utils/format', () => { }, { name: 'testField2', - data_type: DataType.FloatVector, + data_type: 'FloatVector', is_primary_key: false, description: 'Test VECTOR field', dim: 64, @@ -274,7 +290,7 @@ describe('utils/format', () => { element_type: DataType.Int64, }, ], - }; + } as any; const schemaProtoPath = path.resolve( __dirname, @@ -285,6 +301,9 @@ describe('utils/format', () => { const fieldSchemaType = schemaProto.lookupType( 'milvus.proto.schema.FieldSchema' ); + const functionSchemaType = schemaProto.lookupType( + 'milvus.proto.schema.FunctionSchema' + ); const expectedResult = { name: 'testCollection', @@ -295,53 +314,48 @@ describe('utils/format', () => { typeParams: [], indexParams: [], name: 'testField1', - data_type: 5, - is_primary_key: true, description: 'Test PRIMARY KEY field', + data_type: 5, dataType: 5, isPrimaryKey: true, isPartitionKey: false, + isFunctionOutput: false, isClusteringKey: false, }, { - typeParams: [ - { - key: 'dim', - value: '64', - }, - ], + typeParams: [{ key: 'dim', value: '64' }], indexParams: [], name: 'testField2', - data_type: 101, - is_primary_key: false, description: 'Test VECTOR field', + data_type: 'FloatVector', dataType: 101, isPrimaryKey: false, isPartitionKey: false, + isFunctionOutput: false, isClusteringKey: false, }, { - typeParams: [ - { - key: 'max_capacity', - value: '64', - }, - ], + typeParams: [{ key: 'max_capacity', value: '64' }], indexParams: [], name: 'arrayField', - data_type: 22, description: 'Test Array field', - element_type: 5, + data_type: 22, dataType: 22, isPrimaryKey: false, isPartitionKey: false, + isFunctionOutput: false, elementType: 5, + element_type: 5, isClusteringKey: false, }, ], + functions: [], }; - const payload = formatCollectionSchema(data, fieldSchemaType); + const payload = formatCollectionSchema(data, { + fieldSchemaType, + functionSchemaType, + }); expect(payload).toEqual(expectedResult); }); @@ -380,6 +394,11 @@ describe('utils/format', () => { dataType: 101, autoID: false, state: 'created', + is_dynamic: false, + is_clustering_key: false, + is_function_output: false, + nullable: false, + is_partition_key: false, }, { fieldID: '2', @@ -392,12 +411,18 @@ describe('utils/format', () => { dataType: 5, autoID: true, state: 'created', + is_dynamic: false, + is_clustering_key: false, + is_function_output: false, + nullable: false, + is_partition_key: false, }, ], name: 'collection_v8mt0v7x', description: '', enable_dynamic_field: false, autoID: false, + functions: [], }, shards_num: 1, start_positions: [], @@ -408,6 +433,7 @@ describe('utils/format', () => { num_partitions: '0', collection_name: 'test', db_name: '', + functions: [], }; const formatted = formatDescribedCol(response); @@ -600,19 +626,29 @@ describe('utils/format', () => { it('should format search vector correctly', () => { // float vector const floatVector = [1, 2, 3]; - const formattedVector = formatSearchVector( - floatVector, - DataType.FloatVector - ); + const formattedVector = formatSearchData(floatVector, { + dataType: DataType.FloatVector, + } as FieldSchema); expect(formattedVector).toEqual([floatVector]); const floatVectors = [ [1, 2, 3], [4, 5, 6], ]; - expect(formatSearchVector(floatVectors, DataType.FloatVector)).toEqual( - floatVectors - ); + expect( + formatSearchData(floatVectors, { + dataType: DataType.FloatVector, + } as FieldSchema) + ).toEqual(floatVectors); + + // varchar + const varcharVector = 'hello world'; + expect( + formatSearchData(varcharVector, { + dataType: DataType.SparseFloatVector, + is_function_output: true, + } as FieldSchema) + ).toEqual([varcharVector]); }); it('should format sparse vectors correctly', () => { @@ -621,10 +657,9 @@ describe('utils/format', () => { { index: 1, value: 2 }, { index: 3, value: 4 }, ]; - const formattedSparseCooVector = formatSearchVector( - sparseCooVector, - DataType.SparseFloatVector - ); + const formattedSparseCooVector = formatSearchData(sparseCooVector, { + dataType: DataType.SparseFloatVector, + } as FieldSchema); expect(formattedSparseCooVector).toEqual([sparseCooVector]); // sparse csr vector @@ -632,10 +667,9 @@ describe('utils/format', () => { indices: [1, 3], values: [2, 4], }; - const formattedSparseCsrVector = formatSearchVector( - sparseCsrVector, - DataType.SparseFloatVector - ); + const formattedSparseCsrVector = formatSearchData(sparseCsrVector, { + dataType: DataType.SparseFloatVector, + } as FieldSchema); expect(formattedSparseCsrVector).toEqual([sparseCsrVector]); const sparseCsrVectors = [ @@ -648,46 +682,41 @@ describe('utils/format', () => { values: [3, 5], }, ]; - const formattedSparseCsrVectors = formatSearchVector( - sparseCsrVectors, - DataType.SparseFloatVector - ); + const formattedSparseCsrVectors = formatSearchData(sparseCsrVectors, { + dataType: DataType.SparseFloatVector, + } as FieldSchema); expect(formattedSparseCsrVectors).toEqual(sparseCsrVectors); // sparse array vector const sparseArrayVector = [0.1, 0.2, 0.3]; - const formattedSparseArrayVector = formatSearchVector( - sparseArrayVector, - DataType.SparseFloatVector - ); + const formattedSparseArrayVector = formatSearchData(sparseArrayVector, { + dataType: DataType.SparseFloatVector, + } as FieldSchema); expect(formattedSparseArrayVector).toEqual([sparseArrayVector]); const sparseArrayVectors = [ [0.1, 0.2, 0.3], [0.4, 0.5, 0.6], ]; - const formattedSparseArrayVectors = formatSearchVector( - sparseArrayVectors, - DataType.SparseFloatVector - ); + const formattedSparseArrayVectors = formatSearchData(sparseArrayVectors, { + dataType: DataType.SparseFloatVector, + } as FieldSchema); expect(formattedSparseArrayVectors).toEqual(sparseArrayVectors); // sparse dict vector const sparseDictVector = { 1: 2, 3: 4 }; - const formattedSparseDictVector = formatSearchVector( - sparseDictVector, - DataType.SparseFloatVector - ); + const formattedSparseDictVector = formatSearchData(sparseDictVector, { + dataType: DataType.SparseFloatVector, + } as FieldSchema); expect(formattedSparseDictVector).toEqual([sparseDictVector]); const sparseDictVectors = [ { 1: 2, 3: 4 }, { 1: 2, 3: 4 }, ]; - const formattedSparseDictVectors = formatSearchVector( - sparseDictVectors, - DataType.SparseFloatVector - ); + const formattedSparseDictVectors = formatSearchData(sparseDictVectors, { + dataType: DataType.SparseFloatVector, + } as FieldSchema); expect(formattedSparseDictVectors).toEqual(sparseDictVectors); });