From c1e7052dda8b8d30200ae39ef99f3e72d4a0565d Mon Sep 17 00:00:00 2001 From: yhmo Date: Tue, 17 Dec 2024 18:42:13 +0800 Subject: [PATCH] Flatten search params Signed-off-by: yhmo --- .../main/java/io/milvus/param/ParamUtils.java | 104 ++++++++++++++---- .../java/io/milvus/v2/utils/VectorUtils.java | 19 +--- 2 files changed, 90 insertions(+), 33 deletions(-) diff --git a/sdk-core/src/main/java/io/milvus/param/ParamUtils.java b/sdk-core/src/main/java/io/milvus/param/ParamUtils.java index 102781f08..9f74a8d48 100644 --- a/sdk-core/src/main/java/io/milvus/param/ParamUtils.java +++ b/sdk-core/src/main/java/io/milvus/param/ParamUtils.java @@ -766,6 +766,77 @@ public static ByteString convertPlaceholder(List vectors, PlaceholderType pla return placeholderGroup.toByteString(); } + // in versions older than milvus v2.5.1, the search parameters are organized as: + // search_params + // { + // "topk" + // "round_decimal" + // "ignore_growing" + // "offset" + // "metric_type" + // "group_by_field" + // "group_size" + // "strict_group_size" + // + // "params": { + // "nprobe" + // "ef" + // "reorder_k" + // "max_empty_result_buckets" + // "drop_ratio_search" + // "radius" + // "range_filter" + // } + // } + // in milvus v2.5.1, the "params" is removed, all the parameters inside "params" are moved into the top level + // search_params + // { + // "topk" + // "round_decimal" + // "ignore_growing" + // "offset" + // "metric_type" + // "group_by_field" + // "group_size" + // "strict_group_size" + // + // "nprobe" + // "ef" + // "reorder_k" + // "max_empty_result_buckets" + // "drop_ratio_search" + // "radius" + // "range_filter" + // } + // the following logic tries to fit the compatibility between v2.5.1 and older versions + public static void compatibleSearchParams(Map searchParams, SearchRequest.Builder builder) { + Map forOldVersionParams = new HashMap<>(); + List oldParamKeys = Arrays.asList("nprobe", "ef", "reorder_k", "max_empty_result_buckets", "drop_ratio_search", "radius", "range_filter"); + searchParams.forEach((key, value) -> { + // for old versions, these keys are in "params" level + if (oldParamKeys.contains(key)) { + forOldVersionParams.put(key, value); + } + + // for new versions, all keys are in the top level + builder.addSearchParams( + KeyValuePair.newBuilder() + .setKey(key) + .setValue(String.valueOf(value)) + .build()); + }); + try { + String params = JsonUtils.toJson(forOldVersionParams); + builder.addSearchParams( + KeyValuePair.newBuilder() + .setKey(Constant.PARAMS) + .setValue(params) + .build()); + } catch (IllegalArgumentException e) { + throw new MilvusClientException(ErrorCode.INVALID_PARAMS, e.getMessage() + e.getCause().getMessage()); + } + } + @SuppressWarnings("unchecked") public static SearchRequest convertSearchParam(@NonNull SearchParam requestParam) throws ParamException { SearchRequest.Builder builder = SearchRequest.newBuilder() @@ -784,6 +855,19 @@ public static SearchRequest convertSearchParam(@NonNull SearchParam requestParam builder.setNq(requestParam.getNQ()); // search parameters + // tries to fit the compatibility between v2.5.1 and older versions + if (null != requestParam.getParams() && !requestParam.getParams().isEmpty()) { + try { + Map paramMap = JsonUtils.fromJson(requestParam.getParams(), + new TypeToken>() {}.getType()); + compatibleSearchParams(paramMap, builder); + } catch (IllegalArgumentException e) { + throw new ParamException(e.getMessage() + e.getCause().getMessage()); + } + } + + // the following parameters are not changed + // just note: if the searchParams already contains the same key, the following parameters will overwrite it builder.addSearchParams( KeyValuePair.newBuilder() .setKey(Constant.VECTOR_FIELD) @@ -835,26 +919,6 @@ public static SearchRequest convertSearchParam(@NonNull SearchParam requestParam } } - if (null != requestParam.getParams() && !requestParam.getParams().isEmpty()) { - try { - Map paramMap = JsonUtils.fromJson(requestParam.getParams(), - new TypeToken>() {}.getType()); - String offset = paramMap.getOrDefault(Constant.OFFSET, 0).toString(); - builder.addSearchParams( - KeyValuePair.newBuilder() - .setKey(Constant.OFFSET) - .setValue(offset) - .build()); - builder.addSearchParams( - KeyValuePair.newBuilder() - .setKey(Constant.PARAMS) - .setValue(requestParam.getParams()) - .build()); - } catch (IllegalArgumentException e) { - throw new ParamException(e.getMessage() + e.getCause().getMessage()); - } - } - if (!requestParam.getOutFields().isEmpty()) { requestParam.getOutFields().forEach(builder::addOutputFields); } diff --git a/sdk-core/src/main/java/io/milvus/v2/utils/VectorUtils.java b/sdk-core/src/main/java/io/milvus/v2/utils/VectorUtils.java index 53d8bf06e..8e13b7a94 100644 --- a/sdk-core/src/main/java/io/milvus/v2/utils/VectorUtils.java +++ b/sdk-core/src/main/java/io/milvus/v2/utils/VectorUtils.java @@ -161,6 +161,12 @@ public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) { builder.setNq(vectors.size()); // search parameters + // tries to fit the compatibility between v2.5.1 and older versions + Map searchParams = request.getSearchParams(); + ParamUtils.compatibleSearchParams(searchParams, builder); + + // the following parameters are not changed + // just note: if the searchParams already contains the same key, the following parameters will overwrite it if (StringUtils.isNotEmpty(request.getAnnsField())) { builder.addSearchParams( KeyValuePair.newBuilder() @@ -198,19 +204,6 @@ public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) { .build()); } - if (null != request.getSearchParams()) { - try { - String searchParams = JsonUtils.toJson(request.getSearchParams()); - builder.addSearchParams( - KeyValuePair.newBuilder() - .setKey(Constant.PARAMS) - .setValue(searchParams) - .build()); - } catch (IllegalArgumentException e) { - throw new MilvusClientException(ErrorCode.INVALID_PARAMS, e.getMessage() + e.getCause().getMessage()); - } - } - if (request.getGroupByFieldName() != null && !request.getGroupByFieldName().isEmpty()) { builder.addSearchParams( KeyValuePair.newBuilder()