From e7d5304b85d97d0b24f33552481423c6cefcf009 Mon Sep 17 00:00:00 2001 From: Vikasht34 Date: Tue, 4 Mar 2025 16:16:57 -0800 Subject: [PATCH] 3.0.0 Breaking Changes For KNN (#2564) Signed-off-by: Vikasht34 --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/KNNSettings.java | 50 ---------- .../knn/index/engine/SpaceTypeResolver.java | 7 -- .../index/mapper/KNNVectorFieldMapper.java | 2 - .../mapper/KNNVectorFieldMapperUtil.java | 63 ++---------- .../opensearch/knn/index/query/KNNQuery.java | 4 - .../org/opensearch/knn/jni/FaissService.java | 13 --- .../org/opensearch/knn/jni/JNIService.java | 15 --- .../training/FloatTrainingDataConsumer.java | 6 +- .../index/KNNCreateIndexFromModelTests.java | 3 +- .../knn/index/KNNESSettingsTestIT.java | 58 +---------- .../org/opensearch/knn/index/NmslibIT.java | 95 ------------------- .../KNN80DocValuesConsumerTests.java | 2 +- .../knn/index/codec/KNNCodecTestCase.java | 3 +- .../index/engine/SpaceTypeResolverTests.java | 31 ++---- .../mapper/KNNVectorFieldMapperTests.java | 50 ++-------- .../memory/NativeMemoryAllocationTests.java | 2 +- .../knn/index/query/KNNWeightTests.java | 1 - .../opensearch/knn/jni/JNIServiceTests.java | 48 ++++++++-- .../knn/training/TrainingJobTests.java | 4 +- .../org/opensearch/knn/KNNRestTestCase.java | 8 +- 21 files changed, 76 insertions(+), 390 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc8ab03ce1..df6e776586 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Refactoring * Small Refactor Post Lucene 10.0.1 upgrade [#2541](https://github.com/opensearch-project/k-NN/pull/2541) * Refactor codec to leverage backwards_codecs [#2546](https://github.com/opensearch-project/k-NN/pull/2546) +* 3.0.0 Breaking Changes For KNN [#2564] (https://github.com/opensearch-project/k-NN/pull/2564) ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.19...2.x) ### Features diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 3a790a78f6..8de3660d44 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -72,10 +72,7 @@ public class KNNSettings { /** * Settings name */ - public static final String KNN_SPACE_TYPE = "index.knn.space_type"; public static final String INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD = "index.knn.advanced.approximate_threshold"; - public static final String KNN_ALGO_PARAM_M = "index.knn.algo_param.m"; - public static final String KNN_ALGO_PARAM_EF_CONSTRUCTION = "index.knn.algo_param.ef_construction"; public static final String KNN_ALGO_PARAM_EF_SEARCH = "index.knn.algo_param.ef_search"; public static final String KNN_ALGO_PARAM_INDEX_THREAD_QTY = "knn.algo_param.index_thread_qty"; public static final String KNN_MEMORY_CIRCUIT_BREAKER_ENABLED = "knn.memory.circuit_breaker.enabled"; @@ -85,7 +82,6 @@ public class KNNSettings { public static final String KNN_CIRCUIT_BREAKER_TRIGGERED = "knn.circuit_breaker.triggered"; public static final String KNN_CACHE_ITEM_EXPIRY_ENABLED = "knn.cache.item.expiry.enabled"; public static final String KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES = "knn.cache.item.expiry.minutes"; - public static final String KNN_PLUGIN_ENABLED = "knn.plugin.enabled"; public static final String KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE = "knn.circuit_breaker.unset.percentage"; public static final String KNN_INDEX = "index.knn"; public static final String MODEL_INDEX_NUMBER_OF_SHARDS = "knn.model.index.number_of_shards"; @@ -176,14 +172,6 @@ public class KNNSettings { Setting.Property.NodeScope ); - public static final Setting INDEX_KNN_SPACE_TYPE = Setting.simpleString( - KNN_SPACE_TYPE, - INDEX_KNN_DEFAULT_SPACE_TYPE, - new SpaceTypeValidator(), - IndexScope, - Setting.Property.Deprecated - ); - /** * build_vector_data_structure_threshold - This parameter determines when to build vector data structure for knn fields during indexing * and merging. Setting -1 (min) will skip building graph, whereas on any other values, the graph will be built if @@ -199,20 +187,6 @@ public class KNNSettings { Dynamic ); - /** - * M - the number of bi-directional links created for every new element during construction. - * Reasonable range for M is 2-100. Higher M work better on datasets with high intrinsic - * dimensionality and/or high recall, while low M work better for datasets with low intrinsic dimensionality and/or low recalls. - * The parameter also determines the algorithm's memory consumption, which is roughly M * 8-10 bytes per stored element. - */ - public static final Setting INDEX_KNN_ALGO_PARAM_M_SETTING = Setting.intSetting( - KNN_ALGO_PARAM_M, - INDEX_KNN_DEFAULT_ALGO_PARAM_M, - 2, - IndexScope, - Setting.Property.Deprecated - ); - /** * ef or efSearch - the size of the dynamic list for the nearest neighbors (used during the search). * Higher ef leads to more accurate but slower search. ef cannot be set lower than the number of queried nearest neighbors k. @@ -226,18 +200,6 @@ public class KNNSettings { Dynamic ); - /** - * ef_constrution - the parameter has the same meaning as ef, but controls the index_time/index_accuracy. - * Bigger ef_construction leads to longer construction(more indexing time), but better index quality. - */ - public static final Setting INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING = Setting.intSetting( - KNN_ALGO_PARAM_EF_CONSTRUCTION, - INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - 2, - IndexScope, - Setting.Property.Deprecated - ); - public static final Setting MODEL_INDEX_NUMBER_OF_SHARDS_SETTING = Setting.intSetting( MODEL_INDEX_NUMBER_OF_SHARDS, 1, @@ -466,11 +428,6 @@ public class KNNSettings { */ public static Map> dynamicCacheSettings = new HashMap>() { { - /** - * KNN plugin enable/disable setting - */ - put(KNN_PLUGIN_ENABLED, Setting.boolSetting(KNN_PLUGIN_ENABLED, true, NodeScope, Dynamic)); - /** * Weight circuit breaker settings */ @@ -677,10 +634,7 @@ private Setting getSetting(String key) { public List> getSettings() { List> settings = Arrays.asList( - INDEX_KNN_SPACE_TYPE, INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_SETTING, - INDEX_KNN_ALGO_PARAM_M_SETTING, - INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING, INDEX_KNN_ALGO_PARAM_EF_SEARCH_SETTING, KNN_ALGO_PARAM_INDEX_THREAD_QTY_SETTING, KNN_CIRCUIT_BREAKER_TRIGGERED_SETTING, @@ -711,10 +665,6 @@ public List> getSettings() { .collect(Collectors.toList()); } - public static boolean isKNNPluginEnabled() { - return KNNSettings.state().getSettingValue(KNNSettings.KNN_PLUGIN_ENABLED); - } - public static boolean isCircuitBreakerTriggered() { return KNNSettings.state().getSettingValue(KNNSettings.KNN_CIRCUIT_BREAKER_TRIGGERED); } diff --git a/src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java b/src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java index 808bd39b83..31dfb18b51 100644 --- a/src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java +++ b/src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java @@ -8,7 +8,6 @@ import org.apache.logging.log4j.util.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.index.mapper.MapperParsingException; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -49,12 +48,6 @@ public SpaceType resolveSpaceType( // 1. We try to get it from index setting, which is a relic of legacy. // 2. Otherwise, we return a default one. if (isSpaceTypeConfigured(methodSpaceType) == false && isSpaceTypeConfigured(topLevelSpaceType) == false) { - if (indexSettings != null) { - final String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); - if (spaceType != null) { - return SpaceType.getSpace(spaceType); - } - } return getDefaultSpaceType(vectorDataType); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index f1012197fd..0c842f9364 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -60,7 +60,6 @@ import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.useFullFieldNameValidation; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateIfCircuitBreakerIsNotTriggered; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateIfKNNPluginEnabled; import static org.opensearch.knn.index.mapper.ModelFieldMapper.UNSET_MODEL_DIMENSION_IDENTIFIER; /** @@ -686,7 +685,6 @@ protected List getFieldsForByteVector(final byte[] array) { * Validation checks before parsing of doc begins */ protected void validatePreparse() { - validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index d2f9c21b7e..1701154a93 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -33,13 +33,10 @@ import java.util.Arrays; import java.util.Map; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; /** * Utility class for KNNVectorFieldMapper @@ -121,15 +118,6 @@ static void validateIfCircuitBreakerIsNotTriggered() { } } - /** - * Validate if plugin is enabled - */ - static void validateIfKNNPluginEnabled() { - if (!KNNSettings.isKNNPluginEnabled()) { - throw new IllegalStateException("KNN plugin is disabled. To enable update knn.plugin.enabled setting to true"); - } - } - /** * Prerequisite: Index should a knn index which is validated via index settings index.knn setting. This function * assumes that caller has already validated that index is a KNN index. @@ -156,51 +144,12 @@ static boolean useFullFieldNameValidation(final Version indexCreatedVersion) { return indexCreatedVersion != null && indexCreatedVersion.onOrAfter(Version.V_2_17_0); } - public static SpaceType getSpaceType(final Settings indexSettings) { - String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); - if (spaceType == null) { - spaceType = KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; - log.info( - String.format( - "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", - METHOD_PARAMETER_SPACE_TYPE, - spaceType - ) - ); - } - return SpaceType.getSpace(spaceType); - } - - private static int getM(Settings indexSettings) { - String m = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING.getKey()); - if (m == null) { - log.info( - String.format( - "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", - HNSW_ALGO_M, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M - ) - ); - return KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M; - } - return Integer.parseInt(m); + private static int getM() { + return KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M; } - private static int getEfConstruction(Settings indexSettings, Version indexVersion) { - final String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); - if (efConstruction == null) { - final int defaultEFConstructionValue = IndexHyperParametersUtil.getHNSWEFConstructionValue(indexVersion); - log.info( - String.format( - "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. " - + "Picking up default value for the index =%s", - HNSW_ALGO_EF_CONSTRUCTION, - defaultEFConstructionValue - ) - ); - return defaultEFConstructionValue; - } - return Integer.parseInt(efConstruction); + private static int getEfConstruction(Version indexVersion) { + return IndexHyperParametersUtil.getHNSWEFConstructionValue(indexVersion); } static KNNMethodContext createKNNMethodContextFromLegacy( @@ -215,9 +164,9 @@ static KNNMethodContext createKNNMethodContextFromLegacy( METHOD_HNSW, Map.of( METHOD_PARAMETER_M, - KNNVectorFieldMapperUtil.getM(indexSettings), + KNNVectorFieldMapperUtil.getM(), METHOD_PARAMETER_EF_CONSTRUCTION, - KNNVectorFieldMapperUtil.getEfConstruction(indexSettings, indexCreatedVersion) + KNNVectorFieldMapperUtil.getEfConstruction(indexCreatedVersion) ) ) ); diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 3a4201bffd..50ad8ba701 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -21,7 +21,6 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.opensearch.common.StopWatch; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.rescore.RescoreContext; @@ -171,9 +170,6 @@ public KNNQuery filterQuery(Query filterQuery) { */ @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - if (!KNNSettings.isKNNPluginEnabled()) { - throw new IllegalStateException("KNN plugin is disabled. To enable update knn.plugin.enabled to true"); - } StopWatch stopWatch = null; if (log.isDebugEnabled()) { stopWatch = new StopWatch().start(); diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 3ae8bbb926..0376ef19c8 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -407,19 +407,6 @@ public static native KNNQueryResult[] queryBinaryIndexWithFilter( */ public static native byte[] trainByteIndex(Map indexParameters, int dimension, long trainVectorsPointer); - /** - *

- * The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long)} - *

- * Transfer vectors from Java to native - * - * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well - * @param trainingData data to be transferred - * @return pointer to native memory location of training data - */ - @Deprecated(since = "2.14.0", forRemoval = true) - public static native long transferVectors(long vectorsPointer, float[][] trainingData); - /** * Range search index with filter * diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index b490476eb1..fedaf88704 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -419,21 +419,6 @@ public static byte[] trainIndex(Map indexParameters, int dimensi ); } - /** - *

- * The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long, boolean)} - *

- * Transfer vectors from Java to native - * - * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well - * @param trainingData data to be transferred - * @return pointer to native memory location of training data - */ - @Deprecated(since = "2.14.0", forRemoval = true) - public static long transferVectors(long vectorsPointer, float[][] trainingData) { - return FaissService.transferVectors(vectorsPointer, trainingData); - } - /** * Range search index for a given query vector * diff --git a/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java index 2927529456..678d02a5e0 100644 --- a/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java +++ b/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java @@ -15,7 +15,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.jni.JNICommons; -import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.quantization.factory.QuantizerFactory; import org.opensearch.knn.quantization.models.quantizationOutput.BinaryQuantizationOutput; @@ -59,9 +58,10 @@ public void accept(List floats) { } } else { trainingDataAllocation.setMemoryAddress( - JNIService.transferVectors( + JNICommons.storeVectorData( trainingDataAllocation.getMemoryAddress(), - floats.stream().map(v -> ArrayUtils.toPrimitive((Float[]) v)).toArray(float[][]::new) + floats.stream().map(v -> ArrayUtils.toPrimitive((Float[]) v)).toArray(float[][]::new), + floats.size() ) ); } diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index d6ee2d7fde..85567739cf 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -21,6 +21,7 @@ import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; @@ -49,7 +50,7 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException int dimension = 3; // "Train" a faiss flat index - this really just creates an empty index that does brute force k-NN - long vectorsPointer = JNIService.transferVectors(0, new float[0][0]); + long vectorsPointer = JNICommons.storeVectorData(0, new float[0][0], 0); byte[] modelBlob = JNIService.trainIndex( ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "Flat", SPACE_TYPE, spaceType.getValue()), dimension, diff --git a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java index 3687c261d9..4c1522f310 100644 --- a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java @@ -37,11 +37,7 @@ public class KNNESSettingsTestIT extends KNNRestTestCase { public void testKNNLegacySpaceTypeIndexingTest() throws IOException, ParseException { // Configure space_type at index level. This is deprecated and will be removed in the future. - final Settings indexSettings = Settings.builder() - .put("index.knn", true) - .put("knn.algo_param.ef_search", 100) - .put("knn.space_type", SpaceType.INNER_PRODUCT.getValue()) - .build(); + final Settings indexSettings = Settings.builder().put("index.knn", true).put("knn.algo_param.ef_search", 100).build(); // This mapping does not have method. final String testField = "knn_field"; @@ -87,57 +83,7 @@ public void testKNNLegacySpaceTypeIndexingTest() throws IOException, ParseExcept for (KNNResult result : results) { docIds.add(result.getDocId()); } - assertEquals(new HashSet<>(Arrays.asList("1", "3")), docIds); - } - - /** - * KNN Index writes should be blocked when the plugin disabled - * @throws Exception Exception from test - */ - public void testIndexWritesPluginDisabled() throws Exception { - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); - - Float[] vector = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector); - - float[] qvector = { 1.0f, 2.0f }; - Response response = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, qvector, 1), 1); - assertEquals("knn query failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - // disable plugin - updateClusterSettings(KNNSettings.KNN_PLUGIN_ENABLED, false); - - // indexing should be blocked - Exception ex = expectThrows(ResponseException.class, () -> addKnnDoc(INDEX_NAME, "2", FIELD_NAME, vector)); - assertThat(ex.getMessage(), containsString("KNN plugin is disabled")); - - // enable plugin - updateClusterSettings(KNNSettings.KNN_PLUGIN_ENABLED, true); - addKnnDoc(INDEX_NAME, "3", FIELD_NAME, vector); - } - - public void testQueriesPluginDisabled() throws Exception { - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); - - Float[] vector = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector); - - float[] qvector = { 1.0f, 2.0f }; - Response response = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, qvector, 1), 1); - assertEquals("knn query failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - // update settings - updateClusterSettings(KNNSettings.KNN_PLUGIN_ENABLED, false); - - // indexing should be blocked - Exception ex = expectThrows( - ResponseException.class, - () -> searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, qvector, 1), 1) - ); - assertThat(ex.getMessage(), containsString("KNN plugin is disabled")); - // enable plugin - updateClusterSettings(KNNSettings.KNN_PLUGIN_ENABLED, true); - searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, qvector, 1), 1); + assertEquals(new HashSet<>(Arrays.asList("2", "4")), docIds); } public void testItemRemovedFromCache_expiration() throws Exception { diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index e0ba58eb13..640169f1fe 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -305,21 +305,6 @@ public void testDeleteDoc() throws Exception { deleteKnnDoc(INDEX_NAME, "1"); } - public void testCreateIndexWithValidAlgoParams_settings() { - try { - Settings settings = Settings.builder() - .put(getKNNDefaultIndexSettings()) - .put("index.knn.algo_param.m", 32) - .put("index.knn.algo_param.ef_construction", 400) - .build(); - createKnnIndex(INDEX_NAME, settings, createKnnIndexMapping(FIELD_NAME, 2)); - Float[] vector = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector); - } catch (Exception ex) { - fail("Exception not expected as valid index arguements passed: " + ex); - } - } - @SuppressWarnings("unchecked") public void testCreateIndexWithValidAlgoParams_mapping() { try { @@ -373,86 +358,6 @@ public void testCreateIndexWithValidAlgoParams_mapping() { } } - public void testCreateIndexWithValidAlgoParams_mappingAndSettings() { - try { - String spaceType1 = SpaceType.L1.getValue(); - int efConstruction1 = 14; - int m1 = 13; - - Settings settings = Settings.builder() - .put(getKNNDefaultIndexSettings()) - .put("index.knn.algo_param.m", m1) - .put("index.knn.algo_param.ef_construction", efConstruction1) - .build(); - - String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) - .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) - .field(KNNConstants.METHOD_PARAMETER_M, m1) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); - - createKnnIndex(INDEX_NAME + "1", settings, mapping); - Float[] vector = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME + "1", "1", FIELD_NAME, vector); - - String spaceType2 = SpaceType.COSINESIMIL.getValue(); - int efConstruction2 = 114; - int m2 = 113; - - mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME + "1") - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) - .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) - .field(KNNConstants.METHOD_PARAMETER_M, m1) - .endObject() - .endObject() - .endObject() - .startObject(FIELD_NAME + "2") - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType2) - .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction2) - .field(KNNConstants.METHOD_PARAMETER_M, m2) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); - - createKnnIndex(INDEX_NAME + "2", settings, mapping); - addKnnDoc(INDEX_NAME + "2", "1", FIELD_NAME, vector); - } catch (Exception ex) { - fail("Exception not expected as valid index arguments passed: " + ex); - } - } - public void testQueryIndexWithValidQueryAlgoParams() throws IOException { Settings settings = Settings.builder().put(getKNNDefaultIndexSettings()).put("index.knn.algo_param.ef_search", 300).build(); createKnnIndex(INDEX_NAME, settings, createKnnIndexMapping(FIELD_NAME, 2)); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 1d0318bdbc..70bee76d46 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -453,7 +453,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio String modelId = "test-model-id"; float[][] trainingData = TestVectorValues.getRandomVectors(200, dimension); - long trainingPtr = JNIService.transferVectors(0, trainingData); + long trainingPtr = JNICommons.storeVectorData(0, trainingData, trainingData.length * dimension); Map parameters = ImmutableMap.of( INDEX_DESCRIPTION_PARAMETER, diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 18c9e96674..454e85950d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -35,6 +35,7 @@ import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.query.BaseQueryFactory; import org.opensearch.knn.index.query.KNNQueryFactory; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; @@ -224,7 +225,7 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio int dimension = 3; // "Train" a faiss flat index - this really just creates an empty index that does brute force k-NN - long vectorsPointer = JNIService.transferVectors(0, new float[0][0]); + long vectorsPointer = JNICommons.storeVectorData(0, new float[0][0], 0); byte[] modelBlob = JNIService.trainIndex( ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "Flat", SPACE_TYPE, spaceType.getValue()), dimension, diff --git a/src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java index 946dd77865..b29857e76c 100644 --- a/src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java @@ -15,7 +15,6 @@ import static org.opensearch.Version.CURRENT; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; -import static org.opensearch.knn.index.KNNSettings.KNN_SPACE_TYPE; public class SpaceTypeResolverTests extends KNNTestCase { @@ -38,11 +37,7 @@ private void assertResolveSpaceType( public void testResolveSpaceType_whenNoConfigProvided_thenFallbackToVectorDataType() { final Settings emptySettings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); - final Settings settings = Settings.builder() - .put(settings(CURRENT).build()) - .put(KNN_SPACE_TYPE, SpaceType.L2.getValue()) - .put(KNN_INDEX, true) - .build(); + final Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); final KNNMethodContext methodContext = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.COSINESIMIL, MethodComponentContext.EMPTY); final KNNMethodContext emptyMethodContext = new KNNMethodContext( @@ -135,22 +130,16 @@ public void testResolveSpaceType_whenNoConfigProvided_thenFallbackToVectorDataTy "", settings, VectorDataType.BYTE, - SpaceType.getSpace(settings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey())) + SpaceType.getSpace(KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE) ); assertResolveSpaceType( emptyMethodContext, "", settings, VectorDataType.FLOAT, - SpaceType.getSpace(settings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey())) - ); - assertResolveSpaceType( - emptyMethodContext, - "", - settings, - VectorDataType.BINARY, - SpaceType.getSpace(settings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey())) + SpaceType.getSpace(KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE) ); + assertResolveSpaceType(emptyMethodContext, "", settings, VectorDataType.BINARY, SpaceType.getSpace(SpaceType.HAMMING.name())); assertResolveSpaceType(emptyMethodContext, "", emptySettings, VectorDataType.BYTE, SpaceType.DEFAULT); assertResolveSpaceType(emptyMethodContext, "", emptySettings, VectorDataType.FLOAT, SpaceType.DEFAULT); assertResolveSpaceType(emptyMethodContext, "", emptySettings, VectorDataType.BINARY, SpaceType.DEFAULT_BINARY); @@ -183,22 +172,16 @@ public void testResolveSpaceType_whenNoConfigProvided_thenFallbackToVectorDataTy "", settings, VectorDataType.BYTE, - SpaceType.getSpace(settings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey())) + SpaceType.getSpace(KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE) ); assertResolveSpaceType( nullMethodContext, "", settings, VectorDataType.FLOAT, - SpaceType.getSpace(settings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey())) - ); - assertResolveSpaceType( - nullMethodContext, - "", - settings, - VectorDataType.BINARY, - SpaceType.getSpace(settings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey())) + SpaceType.getSpace(KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE) ); + assertResolveSpaceType(nullMethodContext, "", settings, VectorDataType.BINARY, SpaceType.getSpace(SpaceType.HAMMING.name())); assertResolveSpaceType(nullMethodContext, "", emptySettings, VectorDataType.BYTE, SpaceType.DEFAULT); assertResolveSpaceType(nullMethodContext, "", emptySettings, VectorDataType.FLOAT, SpaceType.DEFAULT); assertResolveSpaceType(nullMethodContext, "", emptySettings, VectorDataType.BINARY, SpaceType.DEFAULT_BINARY); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index b0c3d2ffe7..12c739b79b 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -39,7 +39,6 @@ import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; @@ -168,11 +167,7 @@ public void testTypeParser_build_fromKnnMethodContext() throws IOException { .endObject(); // Setup settings - Settings settings = Settings.builder() - .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_ALGO_PARAM_M, mWrong) - .put(KNN_INDEX, true) - .build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( "test-field-name-1", @@ -203,11 +198,7 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro ModelDao modelDao = mock(ModelDao.class); int mForSetting = 71; // Setup settings - Settings settings = Settings.builder() - .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_ALGO_PARAM_M, mForSetting) - .put(KNN_INDEX, true) - .build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); SpaceType methodSpaceType = SpaceType.COSINESIMIL; SpaceType topLevelSpaceType = SpaceType.INNER_PRODUCT; KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -312,7 +303,7 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro ); // this check ensures that legacy mapping is hit, as in legacy mapping we pick M from index settings assertEquals( - mForSetting, + 16, knnVectorFieldMapper.fieldType() .getKnnMappingConfig() .getKnnMethodContext() @@ -365,13 +356,7 @@ public void testBuilder_build_fromModel() { int efConstruction = 17; // Setup settings - Settings settings = Settings.builder() - .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) - .put(KNNSettings.KNN_ALGO_PARAM_M, m) - .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) - .put(KNN_INDEX, true) - .build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); String modelId = "Random modelId"; ModelMetadata mockedModelMetadata = new ModelMetadata( @@ -413,13 +398,7 @@ public void testSpaceType_build_fromLegacy() throws IOException { .field(DIMENSION_FIELD_NAME, 12) .endObject(); - Settings settings = Settings.builder() - .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_ALGO_PARAM_M, m) - .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) - .put(KNNSettings.KNN_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) - .put(KNN_INDEX, true) - .build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( "test-field-name-1", @@ -433,10 +412,7 @@ public void testSpaceType_build_fromLegacy() throws IOException { assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); - assertEquals( - SpaceType.INNER_PRODUCT, - knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType() - ); + assertEquals(SpaceType.L2, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); } public void testBuilder_build_fromLegacy() throws IOException { @@ -452,12 +428,7 @@ public void testBuilder_build_fromLegacy() throws IOException { .field(DIMENSION_FIELD_NAME, 12) .endObject(); - Settings settings = Settings.builder() - .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_ALGO_PARAM_M, m) - .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) - .put(KNN_INDEX, true) - .build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( "test-field-name-1", @@ -923,12 +894,7 @@ public void testTypeParser_parse_fromLegacy() throws IOException { int m = 144; int efConstruction = 123; SpaceType spaceType = SpaceType.L2; - Settings settings = Settings.builder() - .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) - .put(KNNSettings.KNN_ALGO_PARAM_M, m) - .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) - .build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index be20150bcf..7e7cb95071 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -392,7 +392,7 @@ public void testTrainingDataAllocation_close() throws InterruptedException { for (int i = 0; i < numVectors; i++) { Arrays.fill(vectors[i], 1f); } - long memoryAddress = JNIService.transferVectors(0, vectors); + long memoryAddress = JNICommons.storeVectorData(0, vectors, vectors.length * dimension); ExecutorService executorService = Executors.newSingleThreadExecutor(); NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 591c4b1d01..87064f1745 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -147,7 +147,6 @@ public static void setUpClass() throws Exception { ); knnSettingsMockedStatic.when(KNNSettings::getClusterCbLimit).thenReturn(v); knnSettingsMockedStatic.when(KNNSettings::state).thenReturn(knnSettings); - knnSettingsMockedStatic.when(KNNSettings::isKNNPluginEnabled).thenReturn(true); ByteSizeValue cacheSize = ByteSizeValue.parseBytesSizeValue("1024kb", QUANTIZATION_STATE_CACHE_SIZE_LIMIT); // Setting 1MB as an // example when(knnSettings.getSettingValue(eq(QUANTIZATION_STATE_CACHE_SIZE_LIMIT))).thenReturn(cacheSize); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 37c00a104b..a82bc15b36 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -1609,12 +1609,20 @@ public void testFree_faiss_valid() throws IOException { } public void testTransferVectors() { - long trainPointer1 = JNIService.transferVectors(0, testData.indexData.vectors); + long trainPointer1 = JNICommons.storeVectorData( + 0, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertNotEquals(0, trainPointer1); long trainPointer2; for (int i = 0; i < 10; i++) { - trainPointer2 = JNIService.transferVectors(trainPointer1, testData.indexData.vectors); + trainPointer2 = JNICommons.storeVectorData( + trainPointer1, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertEquals(trainPointer1, trainPointer2); } @@ -1720,12 +1728,20 @@ public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException } private long transferVectors(int numDuplicates) { - long trainPointer1 = JNIService.transferVectors(0, testData.indexData.vectors); + long trainPointer1 = JNICommons.storeVectorData( + 0, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertNotEquals(0, trainPointer1); long trainPointer2; for (int i = 0; i < numDuplicates; i++) { - trainPointer2 = JNIService.transferVectors(trainPointer1, testData.indexData.vectors); + trainPointer2 = JNICommons.storeVectorData( + trainPointer1, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertEquals(trainPointer1, trainPointer2); } @@ -1734,12 +1750,20 @@ private long transferVectors(int numDuplicates) { public void createIndexFromTemplate() throws IOException { - long trainPointer1 = JNIService.transferVectors(0, testData.indexData.vectors); + long trainPointer1 = JNICommons.storeVectorData( + 0, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertNotEquals(0, trainPointer1); long trainPointer2; for (int i = 0; i < 10; i++) { - trainPointer2 = JNIService.transferVectors(trainPointer1, testData.indexData.vectors); + trainPointer2 = JNICommons.storeVectorData( + trainPointer1, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertEquals(trainPointer1, trainPointer2); } @@ -1818,7 +1842,11 @@ public void testCreateIndex_whenIOExceptionOccured() { ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()) ); - long trainPointer = JNIService.transferVectors(0, testData.indexData.vectors); + long trainPointer = JNICommons.storeVectorData( + 0, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertNotEquals(0, trainPointer); KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(Version.CURRENT) @@ -1972,7 +2000,11 @@ private void assertQueryResultsMatch(float[][] testQueries, int k, List in private String createFaissIVFPQIndex(Directory directory, int ivfNlist, int pqM, int pqCodeSize, SpaceType spaceType) throws IOException { - long trainPointer = JNIService.transferVectors(0, testData.indexData.vectors); + long trainPointer = JNICommons.storeVectorData( + 0, + testData.indexData.vectors, + testData.indexData.vectors.length * testData.indexData.vectors[0].length + ); assertNotEquals(0, trainPointer); KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(Version.CURRENT) diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index 8db9d67bc6..f005a3711a 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -162,7 +162,7 @@ public void testRun_success() throws IOException, ExecutionException { int tdataPoints = 100; float[][] trainingData = new float[tdataPoints][dimension]; fillFloatArrayRandomly(trainingData); - long memoryAddress = JNIService.transferVectors(0, trainingData); + long memoryAddress = JNICommons.storeVectorData(0, trainingData, trainingData.length * dimension); // Setup model manager NativeMemoryCacheManager nativeMemoryCacheManager = mock(NativeMemoryCacheManager.class); @@ -569,7 +569,7 @@ public void testRun_failure_notEnoughTrainingData() throws ExecutionException { int tdataPoints = 2; float[][] trainingData = new float[tdataPoints][dimension]; fillFloatArrayRandomly(trainingData); - long memoryAddress = JNIService.transferVectors(0, trainingData); + long memoryAddress = JNICommons.storeVectorData(0, trainingData, trainingData.length * dimension); // Setup model manager NativeMemoryCacheManager nativeMemoryCacheManager = mock(NativeMemoryCacheManager.class); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index a8a65c58d1..6996e49934 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1811,13 +1811,7 @@ protected Settings.Builder createKNNIndexCustomLegacyFieldMappingIndexSettingsBu Integer m, Integer ef_construction ) { - return Settings.builder() - .put(NUMBER_OF_SHARDS, 1) - .put(NUMBER_OF_REPLICAS, 0) - .put(INDEX_KNN, true) - .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) - .put(KNNSettings.KNN_ALGO_PARAM_M, m) - .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, ef_construction); + return Settings.builder().put(NUMBER_OF_SHARDS, 1).put(NUMBER_OF_REPLICAS, 0).put(INDEX_KNN, true); } protected Settings createKNNIndexCustomLegacyFieldMappingIndexSettings(SpaceType spaceType, Integer m, Integer ef_construction) {