From 751a55a94cdef4e43b7a9dc485af3c802ec08899 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Tue, 26 Nov 2024 16:10:56 +0800 Subject: [PATCH] fix: function output field should not be inserted as dynamic field (#380) * fix: function output field should not be inserted as dynamic field Signed-off-by: ryjiang * remove console Signed-off-by: ryjiang --------- Signed-off-by: ryjiang --- milvus/grpc/Data.ts | 70 +++++++++++++++++++------------- milvus/utils/Format.ts | 11 +++-- test/grpc/FullTextSearch.spec.ts | 4 +- test/utils/Format.spec.ts | 18 +++++--- 4 files changed, 62 insertions(+), 41 deletions(-) diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index addd82a0..9bb7ba27 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -145,22 +145,27 @@ export class Data extends Collection { // Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`. // If primarykey is set `autoid = true`, you cannot insert the data. // and if function field is set, you need to ignore the field value in the data. + const functionOutputFields: string[] = []; const fieldMap = new Map( - collectionInfo.schema.fields - .filter(v => !v.is_primary_key || !v.autoID) - .filter(v => !v.is_function_output) - .map(v => [ - v.name, - { - name: v.name, - type: v.data_type, // milvus return string here - elementType: v.element_type, - dim: Number(findKeyValue(v.type_params, 'dim')), - data: [], // values container - nullable: v.nullable, - default_value: v.default_value, - }, - ]) + collectionInfo.schema.fields.reduce((acc, v) => { + if (v.is_function_output) { + functionOutputFields.push(v.name); // ignore function field + } else if (!v.is_primary_key || !v.autoID) { + acc.push([ + v.name, + { + name: v.name, + type: v.data_type, // milvus return string here + elementType: v.element_type, + dim: Number(findKeyValue(v.type_params, 'dim')), + data: [], // values container + nullable: v.nullable, + default_value: v.default_value, + }, + ]); + } + return acc; + }, [] as [string, _Field][]) ); // dynamic field is enabled, create $meta field @@ -179,7 +184,12 @@ export class Data extends Collection { data.fields_data.forEach((rowData, rowIndex) => { // if support dynamic field, all field not in the schema would be grouped to a dynamic field rowData = isDynamic - ? buildDynamicRow(rowData, fieldMap, DEFAULT_DYNAMIC_FIELD) + ? buildDynamicRow( + rowData, + fieldMap, + DEFAULT_DYNAMIC_FIELD, + functionOutputFields + ) : rowData; // get each fieldname from the row object @@ -230,10 +240,18 @@ export class Data extends Collection { const dataKey = getDataKey(type); const elementType = DataTypeMap[field.elementType!]; const elementTypeKey = getDataKey(elementType); - const valid_data = getValidDataArray( - field.data, - data.fields_data?.length! - ); + + // check if need valid data + const needValidData = + key !== 'vectors' && + (field.nullable === true || + (typeof field.default_value !== 'undefined' && + field.default_value !== null)); + + // get valid data + const valid_data = needValidData + ? getValidDataArray(field.data, data.fields_data?.length!) + : []; // build key value let keyValue; @@ -295,18 +313,12 @@ export class Data extends Collection { break; } - const needValidData = - key !== 'vectors' && - (field.nullable === true || - (typeof field.default_value !== 'undefined' && - field.default_value !== null)); - return { type, field_name: field.name, is_dynamic: field.name === DEFAULT_DYNAMIC_FIELD, [key]: keyValue, - valid_data: needValidData ? valid_data : [], + valid_data: valid_data, }; }); @@ -478,7 +490,9 @@ export class Data extends Collection { describeCollectionRequest.db_name = data.db_name; } - const collectionInfo = await this.describeCollection(describeCollectionRequest); + const collectionInfo = await this.describeCollection( + describeCollectionRequest + ); // build search params const { request, nq, round_decimal, isHybridSearch } = buildSearchRequest( diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index 1fa94855..6e614029 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -429,7 +429,8 @@ export const formatDescribedCol = ( export const buildDynamicRow = ( rowData: RowData, fieldMap: Map, - dynamicFieldName: string + dynamicFieldName: string, + functionOutputFields: string[] ) => { const originRow = cloneObj(rowData); @@ -443,9 +444,11 @@ export const buildDynamicRow = ( // if the key is in the fieldMap, add it to the non-dynamic fields row[key] = originRow[key]; } else { - const obj: JSON = row[dynamicFieldName] as JSON; - // otherwise, add it to the dynamic field - obj[key] = originRow[key]; + if (!functionOutputFields.includes(key)) { + const obj: JSON = row[dynamicFieldName] as JSON; + // otherwise, add it to the dynamic field + obj[key] = originRow[key]; + } } } diff --git a/test/grpc/FullTextSearch.spec.ts b/test/grpc/FullTextSearch.spec.ts index 635625c0..3560d555 100644 --- a/test/grpc/FullTextSearch.spec.ts +++ b/test/grpc/FullTextSearch.spec.ts @@ -14,7 +14,7 @@ import { dynamicFields, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION = GENERATE_NAME(); const dbParam = { db_name: 'FullTextSearch', @@ -252,8 +252,6 @@ describe(`FulltextSearch API`, () => { consistency_level: ConsistencyLevelEnum.Strong, }); - console.dir(search3, { depth: null }); - expect(search3.status.error_code).toEqual(ErrorCode.SUCCESS); }); }); diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index 66c794c4..756529da 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -488,15 +488,21 @@ describe('utils/format', () => { const data = {}; const fieldsDataMap = new Map(); const dynamicField = 'dynamic'; - const result = buildDynamicRow(data, fieldsDataMap, dynamicField); + const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []); expect(result).toEqual({}); }); it('should return an object with dynamicField key when all data contains keys not in fieldsDataMap', () => { - const data = { key: 'value' }; + const data = { key: 'value', key2: 'value2' }; const fieldsDataMap = new Map(); const dynamicField = 'dynamic'; - const result = buildDynamicRow(data, fieldsDataMap, dynamicField); + const ignoreFields = ['key2']; + const result = buildDynamicRow( + data, + fieldsDataMap, + dynamicField, + ignoreFields + ); expect(result).toEqual({ [dynamicField]: { key: 'value' } }); }); @@ -513,7 +519,7 @@ describe('utils/format', () => { ], ]); const dynamicField = 'dynamic'; - const result = buildDynamicRow(data, fieldsDataMap, dynamicField); + const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []); expect(result).toEqual({ key1: 'value1', [dynamicField]: { key2: 'value2' }, @@ -541,7 +547,7 @@ describe('utils/format', () => { ], ]); const dynamicField = 'dynamic'; - const result = buildDynamicRow(data, fieldsDataMap, dynamicField); + const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []); expect(result).toEqual({ [dynamicField]: {}, key1: 'value1', @@ -562,7 +568,7 @@ describe('utils/format', () => { ], ]); const dynamicField = 'dynamic'; - const result = buildDynamicRow(data, fieldsDataMap, dynamicField); + const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []); expect(result).toEqual({ key1: 'value1', [dynamicField]: { key2: 'value2' },