[2.5] support default value and null (#369)

* init 2.5 Signed-off-by: ryjiang <[email protected]> * update milvus version Signed-off-by: ryjiang <[email protected]> * update proto Signed-off-by: ryjiang <[email protected]> * WIP: support default value & null Signed-off-by: ryjiang <[email protected]> * fix json format Signed-off-by: ryjiang <[email protected]> * WIP Signed-off-by: ryjiang <[email protected]> * WIP Signed-off-by: ryjiang <[email protected]> * WIP Signed-off-by: ryjiang <[email protected]> * add null test for array Signed-off-by: ryjiang <[email protected]> * recover ip Signed-off-by: ryjiang <[email protected]> * WIP Signed-off-by: shanghaikid <[email protected]> * finish default_null Signed-off-by: ryjiang <[email protected]> * update test version Signed-off-by: ryjiang <[email protected]> * fix test Signed-off-by: ryjiang <[email protected]> * update test Signed-off-by: ryjiang <[email protected]> --------- Signed-off-by: ryjiang <[email protected]> Signed-off-by: shanghaikid <[email protected]>
milvus-io · Nov 6, 2024 · 9f46da2 · 9f46da2
1 parent f546023
commit 9f46da2
Show file tree

Hide file tree

Showing 20 changed files with 255 additions and 81 deletions.
diff --git a/milvus/grpc/Collection.ts b/milvus/grpc/Collection.ts
@@ -54,6 +54,7 @@ import {
   CreateCollectionWithFieldsReq,
   CreateCollectionWithSchemaReq,
   FieldSchema,
+  isVectorType,
 } from '../';
 
 /**
@@ -1054,9 +1055,12 @@ export class Collection extends Database {
    *  });
    * ```
    */
-  async getPkFieldName(data: DescribeCollectionReq): Promise<string> {
+  async getPkFieldName(
+    data: DescribeCollectionReq,
+    desc?: DescribeCollectionResponse
+  ): Promise<string> {
     // get collection info
-    const collectionInfo = await this.describeCollection(data);
+    const collectionInfo = desc ? desc : await this.describeCollection(data);
 
     // pk field
     let pkField = '';
@@ -1094,10 +1098,11 @@ export class Collection extends Database {
    * ```
    */
   async getPkFieldType(
-    data: DescribeCollectionReq
+    data: DescribeCollectionReq,
+    desc?: DescribeCollectionResponse
   ): Promise<keyof typeof DataType> {
     // get collection info
-    const collectionInfo = await this.describeCollection(data);
+    const collectionInfo = desc ? desc : await this.describeCollection(data);
 
     // pk field type
     let pkFieldType: keyof typeof DataType = 'Int64';

diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts
@@ -48,7 +48,7 @@ import {
   buildDynamicRow,
   buildFieldDataMap,
   getDataKey,
-  Field,
+  _Field,
   buildFieldData,
   BinaryVector,
   RowData,
@@ -65,6 +65,7 @@ import {
   sparseRowsToBytes,
   getSparseDim,
   f32ArrayToBinaryBytes,
+  getValidDataArray,
   NO_LIMIT,
 } from '../';
 import { Collection } from './Collection';
@@ -143,7 +144,7 @@ export class Data extends Collection {
     // Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`.
     // If primarykey is set `autoid = true`, you cannot insert the data.
     // and if function field is set, you need to ignore the field value in the data.
-    const fieldMap = new Map<string, Field>(
+    const fieldMap = new Map<string, _Field>(
       collectionInfo.schema.fields
         .filter(v => !v.is_primary_key || !v.autoID)
         .filter(v => !v.is_function_output)
@@ -155,6 +156,8 @@ export class Data extends Collection {
             elementType: v.element_type,
             dim: Number(findKeyValue(v.type_params, 'dim')),
             data: [], // values container
+            nullable: v.nullable,
+            default_value: v.default_value,
           },
         ])
     );
@@ -167,6 +170,7 @@ export class Data extends Collection {
         type: 'JSON',
         elementType: 'None',
         data: [], // value container
+        nullable: false,
       });
     }
 
@@ -225,6 +229,10 @@ export class Data extends Collection {
       const dataKey = getDataKey(type);
       const elementType = DataTypeMap[field.elementType!];
       const elementTypeKey = getDataKey(elementType);
+      const valid_data = getValidDataArray(
+        field.data,
+        data.fields_data?.length!
+      );
 
       // build key value
       let keyValue;
@@ -263,32 +271,41 @@ export class Data extends Collection {
         case DataType.Array:
           keyValue = {
             [dataKey]: {
-              data: field.data.map(d => {
-                return {
-                  [elementTypeKey]: {
-                    type: elementType,
-                    data: d,
-                  },
-                };
-              }),
+              data: field.data
+                .filter(v => v !== undefined)
+                .map(d => {
+                  return {
+                    [elementTypeKey]: {
+                      type: elementType,
+                      data: d,
+                    },
+                  };
+                }),
               element_type: elementType,
             },
           };
           break;
         default:
           keyValue = {
             [dataKey]: {
-              data: field.data,
+              data: field.data.filter(v => v !== undefined),
             },
           };
           break;
       }
 
+      const needValidData =
+        key !== 'vectors' &&
+        (field.nullable === true ||
+          (typeof field.default_value !== 'undefined' &&
+            field.default_value !== null));
+
       return {
         type,
         field_name: field.name,
         is_dynamic: field.name === DEFAULT_DYNAMIC_FIELD,
         [key]: keyValue,
+        valid_data: needValidData ? valid_data : [],
       };
     });
 
@@ -931,6 +948,7 @@ export class Data extends Collection {
     // always get output_fields from fields_data
     const output_fields = promise.fields_data.map(f => f.field_name);
 
+    // build field data map
     const fieldsDataMap = buildFieldDataMap(
       promise.fields_data,
       data.transformers

diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts
@@ -30,11 +30,11 @@ export interface FieldSchema {
   element_type?: keyof typeof DataType;
   default_value?: number | string;
   dataType: DataType;
-  is_partition_key: boolean;
-  is_dynamic: boolean;
-  is_clustering_key: boolean;
+  is_partition_key?: boolean;
+  is_dynamic?: boolean;
+  is_clustering_key?: boolean;
+  nullable?: boolean;
   is_function_output: boolean;
-  nullable: boolean;
 }
 
 export interface CollectionData {
@@ -80,6 +80,7 @@ export interface FieldType {
   max_capacity?: TypeParam;
   max_length?: TypeParam;
   default_value?: number | string;
+  nullable?: boolean;
   enable_match?: boolean;
   tokenizer_params?: Record<string, any>;
   enable_tokenizer?: boolean;

diff --git a/milvus/types/Data.ts b/milvus/types/Data.ts
@@ -72,19 +72,22 @@ export type FieldData =
   | VarChar
   | JSON
   | Array
-  | VectorTypes;
+  | VectorTypes
+  | null;
 
 // Represents a row of data in Milvus.
 export interface RowData {
   [x: string]: FieldData;
 }
 
-export interface Field {
+export interface _Field {
   name: string;
   type: keyof typeof DataType;
   elementType?: keyof typeof DataType;
   data: FieldData[];
   dim?: number;
+  nullable?: boolean;
+  default_value?: FieldData;
 }
 
 export interface FlushReq extends GrpcTimeOut {
@@ -455,6 +458,8 @@ export interface QueryRes extends resStatusResponse {
       [x: string]: any;
       data: string;
     };
+    is_dynamic: boolean;
+    valid_data: boolean[];
   }[];
   output_fields: string[];
   collection_name: string;

diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts
@@ -10,7 +10,7 @@ import {
   DescribeCollectionResponse,
   getDataKey,
   RowData,
-  Field,
+  _Field,
   JSON,
   FieldData,
   CreateCollectionWithFieldsReq,
@@ -421,7 +421,7 @@ export const formatDescribedCol = (
  */
 export const buildDynamicRow = (
   rowData: RowData,
-  fieldMap: Map<string, Field>,
+  fieldMap: Map<string, _Field>,
   dynamicFieldName: string
 ) => {
   const originRow = cloneObj(rowData);
@@ -537,17 +537,27 @@ export const buildFieldDataMap = (
         });
       }
 
-      // decode json
       switch (dataKey) {
+        // decode json
         case 'json_data':
           field_data.forEach((buffer: any, i: number) => {
-            // console.log(JSON.parse(buffer.toString()));
-            field_data[i] = JSON.parse(buffer.toString());
+            field_data[i] = buffer.length
+              ? JSON.parse(buffer.toString())
+              : null;
           });
           break;
         default:
           break;
       }
+
+      // set the field data with null if item.valid_data is not empty array, it the item in valid_data is false, set the field data with null
+      if (item.valid_data && item.valid_data.length) {
+        item.valid_data.forEach((v: any, i: number) => {
+          if (!v) {
+            field_data[i] = null;
+          }
+        });
+      }
     }
 
     // Add the parsed data to the fieldsDataMap
@@ -591,7 +601,7 @@ export const getAuthString = (data: {
  */
 export const buildFieldData = (
   rowData: RowData,
-  field: Field,
+  field: _Field,
   transformers?: InsertTransformers
 ): FieldData => {
   const { type, elementType, name } = field;
@@ -614,7 +624,9 @@ export const buildFieldData = (
         ? f16Transformer(rowData[name] as Float16Vector)
         : rowData[name];
     case DataType.JSON:
-      return Buffer.from(JSON.stringify(rowData[name] || {}));
+      return rowData[name]
+        ? Buffer.from(JSON.stringify(rowData[name] || {}))
+        : Buffer.alloc(0);
     case DataType.Array:
       const elementField = { ...field, type: elementType! };
       return buildFieldData(rowData, elementField, transformers);

diff --git a/milvus/utils/Function.ts b/milvus/utils/Function.ts
@@ -7,6 +7,7 @@ import {
   DEFAULT_MIN_INT64,
   SearchResultData,
   SparseFloatVector,
+  FieldData,
 } from '../';
 import { Pool } from 'generic-pool';
 
@@ -236,3 +237,11 @@ export const getSparseDim = (data: SparseFloatVector[]) => {
   }
   return dim;
 };
+
+// get valid data
+// create a length array with valid data, if the data is undefined or null, return false, otherwise return true
+export const getValidDataArray = (data: FieldData[], length: number) => {
+  return Array.from({ length }).map((_, i) => {
+    return data[i] !== undefined && data[i] !== null;
+  });
+};
diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts
@@ -200,7 +200,7 @@ export const getRetryInterceptor = ({
               logger.debug(
                 `\x1b[32m[Response(${
                   Date.now() - startTime.getTime()
-                }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${string}`
+                }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${msg}`
               );
 
               savedMessageNext(savedReceiveMessage);
@@ -217,7 +217,7 @@ export const getRetryInterceptor = ({
         const msg =
           string.length > 2048 ? string.slice(0, 2048) + '...' : string;
         logger.debug(
-          `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${string}`
+          `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}`
         );
         savedSendMessage = message;
         next(message);

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@zilliz/milvus2-sdk-node",
   "author": "[email protected]",
-  "milvusVersion": "master-20241024-f78f6112-amd64",
+  "milvusVersion": "master-20241105-bd04cac4-amd64",
   "version": "2.4.9",
   "main": "dist/milvus",
   "files": [

diff --git a/proto b/proto
diff --git a/test/build/Collection.spec.ts b/test/build/Collection.spec.ts
@@ -91,7 +91,11 @@ describe('Collection Api', () => {
 
     try {
       await milvusClient.createCollection(
-        genCollectionParams({ collectionName: 'any', dim: [8] })
+        genCollectionParams({
+          collectionName: 'any',
+          vectorType: [DataType.BinaryVector],
+          dim: [6],
+        })
       );
     } catch (error) {
       expect(error.message).toEqual(
+235 −231		go-api/commonpb/common.pb.go
+2,617 −2,508		go-api/milvuspb/milvus.pb.go
+134 −51		go-api/rgpb/rg.pb.go
+511 −166		go-api/schemapb/schema.pb.go
+1 −0		proto/common.proto
+6 −2		proto/milvus.proto
+7 −0		proto/rg.proto
+22 −1		proto/schema.proto