From 872caa4e2cb6221e45f213702ec14d9be5b5a922 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 23 Jan 2025 17:59:24 +0000 Subject: [PATCH] Deprecate nmslib engine Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 1 + .../java/org/opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/engine/KNNEngine.java | 10 ++++++++++ .../opensearch/knn/index/engine/nmslib/Nmslib.java | 4 ++++ .../knn/index/engine/nmslib/NmslibHNSWMethod.java | 4 ++++ .../knn/index/engine/nmslib/NmslibMethodResolver.java | 10 ++++++++++ src/main/java/org/opensearch/knn/jni/JNIService.java | 11 +++++++++++ .../java/org/opensearch/knn/jni/NmslibService.java | 4 ++++ src/test/java/org/opensearch/knn/index/NmslibIT.java | 1 + .../engine/nmslib/NmslibMethodResolverTests.java | 1 + 10 files changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ffbfbead..78816046dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,4 +57,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Bump Faiss commit from 1f42e81 to 0cbc2a8 to accelerate hamming distance calculation using _mm512_popcnt_epi64 intrinsic and also add avx512-fp16 instructions to boost performance [#2381](https://github.com/opensearch-project/k-NN/pull/2381) * Enabled indices.breaker.total.use_real_memory setting via build.gradle for integTest Cluster to catch heap CB in local ITs and github CI actions [#2395](https://github.com/opensearch-project/k-NN/pull/2395/) * Fixing Lucene912Codec Issue with BWC for Lucene 10.0.1 upgrade[#2429](https://github.com/opensearch-project/k-NN/pull/2429) +* Deprecate nmslib engine (#2427)[https://github.com/opensearch-project/k-NN/pull/2427] ### Refactoring diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 4479099e82..cfc79a08e3 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -91,6 +91,7 @@ public class KNNConstants { public static final int LUCENE_SQ_DEFAULT_BITS = 7; // nmslib specific constants + @Deprecated(since = "2.19.0", forRemoval = true) public static final String NMSLIB_NAME = "nmslib"; public static final String COMMONS_NAME = "common"; public static final String SPACE_TYPE = "spaceType"; // used as field info key diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index 1e560a11ba..ea06d985f8 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -6,6 +6,8 @@ package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableSet; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.faiss.Faiss; @@ -25,6 +27,7 @@ * passed to the respective k-NN library's JNI layer. */ public enum KNNEngine implements KNNLibrary { + @Deprecated(since = "2.19.0", forRemoval = true) NMSLIB(NMSLIB_NAME, Nmslib.INSTANCE), FAISS(FAISS_NAME, Faiss.INSTANCE), LUCENE(LUCENE_NAME, Lucene.INSTANCE); @@ -34,6 +37,7 @@ public enum KNNEngine implements KNNLibrary { private static final Set CUSTOM_SEGMENT_FILE_ENGINES = ImmutableSet.of(KNNEngine.NMSLIB, KNNEngine.FAISS); private static final Set ENGINES_SUPPORTING_FILTERS = ImmutableSet.of(KNNEngine.LUCENE, KNNEngine.FAISS); public static final Set ENGINES_SUPPORTING_RADIAL_SEARCH = ImmutableSet.of(KNNEngine.LUCENE, KNNEngine.FAISS); + private static Logger logger = LogManager.getLogger(KNNEngine.class); private static Map MAX_DIMENSIONS_BY_ENGINE = Map.of( KNNEngine.NMSLIB, @@ -66,6 +70,9 @@ public enum KNNEngine implements KNNLibrary { */ public static KNNEngine getEngine(String name) { if (NMSLIB.getName().equalsIgnoreCase(name)) { + logger.warn( + "[Deprecation] nmslib engine is deprecated and will be removed in a future release. Please use Faiss or Lucene engine instead." + ); return NMSLIB; } @@ -88,6 +95,9 @@ public static KNNEngine getEngine(String name) { */ public static KNNEngine getEngineNameFromPath(String path) { if (path.endsWith(KNNEngine.NMSLIB.getExtension()) || path.endsWith(KNNEngine.NMSLIB.getCompoundExtension())) { + logger.warn( + "[Deprecation] nmslib engine is deprecated and will be removed in a future release. Please use Faiss or Lucene engine instead." + ); return KNNEngine.NMSLIB; } diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java index 4d7f7f4237..869f851e97 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java @@ -22,7 +22,11 @@ /** * Implements NativeLibrary for the nmslib native library + * + * @deprecated As of 2.19.0, please use {@link org.opensearch.knn.index.engine.faiss.Faiss} or Lucene's native k-NN. + * This engine will be removed in a future release. */ +@Deprecated(since = "2.19.0", forRemoval = true) public class Nmslib extends NativeLibrary { // Extension to be used for Nmslib files. It is ".hnsw" and not ".nmslib" for legacy purposes. public final static String EXTENSION = ".hnsw"; diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java index d2440926ef..447972c84a 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java @@ -24,7 +24,11 @@ /** * Nmslib's HNSW implementation + * + * @deprecated As of 2.19.0, please use {@link org.opensearch.knn.index.engine.faiss.Faiss} or Lucene engine. + * This engine will be removed in a future release. */ +@Deprecated(since = "2.19.0", forRemoval = true) public class NmslibHNSWMethod extends AbstractKNNMethod { private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java index 619a00edab..ef29b7686b 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java @@ -5,6 +5,8 @@ package org.opensearch.knn.index.engine.nmslib; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.AbstractMethodResolver; @@ -23,10 +25,15 @@ /** * Method resolution logic for nmslib. Because nmslib does not support quantization, it is in general a validation * before returning the original request + * + * @deprecated As of 2.19.0, please use {@link org.opensearch.knn.index.engine.faiss.Faiss} or Lucene engine. + * This engine will be removed in a future release. */ +@Deprecated(since = "2.19.0", forRemoval = true) public class NmslibMethodResolver extends AbstractMethodResolver { private static final Set SUPPORTED_COMPRESSION_LEVELS = Set.of(CompressionLevel.x1); + private static Logger logger = LogManager.getLogger(NmslibMethodResolver.class); @Override public ResolvedMethodContext resolveMethod( @@ -35,6 +42,9 @@ public ResolvedMethodContext resolveMethod( boolean shouldRequireTraining, final SpaceType spaceType ) { + logger.warn( + "[Deprecation] nmslib engine is deprecated and will be removed in a future release. Please use Faiss or Lucene engine instead." + ); validateConfig(knnMethodConfigContext, shouldRequireTraining); KNNMethodContext resolvedKNNMethodContext = initResolvedKNNMethodContext( knnMethodContext, diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index b490476eb1..3d2a13f97f 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -12,6 +12,8 @@ package org.opensearch.knn.jni; import org.apache.commons.lang.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.common.Nullable; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNEngine; @@ -27,6 +29,9 @@ * Service to distribute requests to the proper engine jni service */ public class JNIService { + + private static Logger logger = LogManager.getLogger(JNIService.class); + /** * Initialize an index for the native library. Takes in numDocs to * allocate the correct amount of memory. @@ -137,6 +142,9 @@ public static void createIndex( KNNEngine knnEngine ) { if (KNNEngine.NMSLIB == knnEngine) { + logger.warn( + "[Deprecation] nmslib engine is deprecated and will be removed in a future release. Please use Faiss or Lucene engine instead." + ); NmslibService.createIndex(ids, vectorsAddress, dim, output, parameters); return; } @@ -201,6 +209,9 @@ public static long loadIndex(IndexInputWithBuffer readStream, Map