Milvus-model as pymilvus submodule (#61)

Signed-off-by: junjie.jiang <[email protected]>

pyproject.toml

@@ -30,6 +30,12 @@ classifiers = [
 
 dynamic = ["version"]
 
+[tool.setuptools]
+package-dir = { "" = "src/" }
+
+[tool.setuptools.package-data]
+"pymilvus.model.sparse.bm25" = ["lang.yaml"]
+
 [project.urls]
 repository = "https://github.com/milvus-io/milvus-model"  # Update the repository URL
 

milvus_model/dense/__init__.py → src/pymilvus/model/dense/__init__.py

@@ -10,19 +10,19 @@
     "InstructorEmbeddingFunction"
 ]
 
-from milvus_model.utils.lazy_import import LazyImport
+from pymilvus.model.utils.lazy_import import LazyImport
 
-jinaai = LazyImport("jinaai", globals(), "milvus_model.dense.jinaai")
-openai = LazyImport("openai", globals(), "milvus_model.dense.openai")
+jinaai = LazyImport("jinaai", globals(), "pymilvus.model.dense.jinaai")
+openai = LazyImport("openai", globals(), "pymilvus.model.dense.openai")
 sentence_transformer = LazyImport(
-    "sentence_transformer", globals(), "milvus_model.dense.sentence_transformer"
+    "sentence_transformer", globals(), "pymilvus.model.dense.sentence_transformer"
 )
-voyageai = LazyImport("voyageai", globals(), "milvus_model.dense.voyageai")
-onnx = LazyImport("onnx", globals(), "milvus_model.dense.onnx")
-cohere = LazyImport("cohere", globals(), "milvus_model.dense.cohere")
-mistralai = LazyImport("mistralai", globals(), "milvus_model.dense.mistralai")
-nomic = LazyImport("nomic", globals(), "milvus_model.dense.nomic")
-instructor = LazyImport("instructor", globals(), "milvus_model.dense.instructor")
+voyageai = LazyImport("voyageai", globals(), "pymilvus.model.dense.voyageai")
+onnx = LazyImport("onnx", globals(), "pymilvus.model.dense.onnx")
+cohere = LazyImport("cohere", globals(), "pymilvus.model.dense.cohere")
+mistralai = LazyImport("mistralai", globals(), "pymilvus.model.dense.mistralai")
+nomic = LazyImport("nomic", globals(), "pymilvus.model.dense.nomic")
+instructor = LazyImport("instructor", globals(), "pymilvus.model.dense.instructor")
 
 
 def JinaEmbeddingFunction(*args, **kwargs):

milvus_model/dense/cohere.py → src/pymilvus/model/dense/cohere.py

@@ -3,8 +3,8 @@
 from collections import defaultdict
 import numpy as np
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_cohere
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_cohere
 
 import_cohere()
 import cohere

milvus_model/dense/instructor.py → src/pymilvus/model/dense/instructor.py

@@ -1,8 +1,8 @@
 from typing import List, Optional
 import numpy as np
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_sentence_transformers, import_huggingface_hub
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_sentence_transformers, import_huggingface_hub
 
 import_sentence_transformers()
 import_huggingface_hub()

milvus_model/dense/jinaai.py → src/pymilvus/model/dense/jinaai.py

@@ -4,7 +4,7 @@
 import numpy as np
 import requests
 
-from milvus_model.base import BaseEmbeddingFunction
+from pymilvus.model.base import BaseEmbeddingFunction
 
 API_URL = "https://api.jina.ai/v1/embeddings"
 

milvus_model/dense/mistralai.py → src/pymilvus/model/dense/mistralai.py

@@ -3,8 +3,8 @@
 import numpy as np
 from collections import defaultdict
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_mistralai
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_mistralai
 
 import_mistralai()
 from mistralai import Mistral

milvus_model/dense/nomic.py → src/pymilvus/model/dense/nomic.py

@@ -3,8 +3,8 @@
 import os
 from collections import defaultdict
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_nomic
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_nomic
 
 import_nomic()
 from nomic import embed

milvus_model/dense/onnx.py → src/pymilvus/model/dense/onnx.py

@@ -6,7 +6,7 @@
 import numpy as np
 from typing import List
 
-from milvus_model.base import BaseEmbeddingFunction
+from pymilvus.model.base import BaseEmbeddingFunction
 
 class OnnxEmbeddingFunction(BaseEmbeddingFunction):
     def __init__(self, model_name: str = "GPTCache/paraphrase-albert-onnx", tokenizer_name: str = "GPTCache/paraphrase-albert-small-v2"):

milvus_model/dense/openai.py → src/pymilvus/model/dense/openai.py

@@ -3,8 +3,8 @@
 
 import numpy as np
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_openai
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_openai
 
 import_openai()
 from openai import OpenAI

milvus_model/dense/sentence_transformer.py → src/pymilvus/model/dense/sentence_transformer.py

@@ -2,8 +2,8 @@
 
 import numpy as np
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_sentence_transformers
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_sentence_transformers
 
 import_sentence_transformers()
 from sentence_transformers import SentenceTransformer

milvus_model/dense/voyageai.py → src/pymilvus/model/dense/voyageai.py

@@ -4,8 +4,8 @@
 import numpy as np
 import struct
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_voyageai
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_voyageai
 
 import_voyageai()
 import voyageai

milvus_model/hybrid/__init__.py → src/pymilvus/model/hybrid/__init__.py

@@ -1,9 +1,9 @@
 __all__ = ["BGEM3EmbeddingFunction", "MGTEEmbeddingFunction"]
 
-from milvus_model.utils.lazy_import import LazyImport
+from pymilvus.model.utils.lazy_import import LazyImport
 
-bge_m3 = LazyImport("bge_m3", globals(), "milvus_model.hybrid.bge_m3")
-mgte = LazyImport("mgte", globals(), "milvus_model.hybrid.mgte")
+bge_m3 = LazyImport("bge_m3", globals(), "pymilvus.model.hybrid.bge_m3")
+mgte = LazyImport("mgte", globals(), "pymilvus.model.hybrid.mgte")
 
 def BGEM3EmbeddingFunction(*args, **kwargs):
     return bge_m3.BGEM3EmbeddingFunction(*args, **kwargs)

milvus_model/hybrid/bge_m3.py → src/pymilvus/model/hybrid/bge_m3.py

@@ -4,9 +4,9 @@
 from scipy.sparse import csr_array
 import numpy as np
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_FlagEmbedding, import_datasets
-from milvus_model.sparse.utils import stack_sparse_embeddings
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_FlagEmbedding, import_datasets
+from pymilvus.model.sparse.utils import stack_sparse_embeddings
 
 import_datasets()
 import_FlagEmbedding()

milvus_model/hybrid/mgte.py → src/pymilvus/model/hybrid/mgte.py

@@ -11,9 +11,9 @@
 
 import numpy as np
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_transformers, import_scipy, import_torch
-from milvus_model.sparse.utils import stack_sparse_embeddings
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_transformers, import_scipy, import_torch
+from pymilvus.model.sparse.utils import stack_sparse_embeddings
 
 import_torch()
 import_scipy()

milvus_model/reranker/__init__.py → src/pymilvus/model/reranker/__init__.py

@@ -6,13 +6,13 @@
     "JinaRerankFunction",
 ]
 
-from milvus_model.utils.lazy_import import LazyImport
+from pymilvus.model.utils.lazy_import import LazyImport
 
-bgegreranker = LazyImport("bgereranker", globals(), "milvus_model.reranker.bgereranker")
-cohere = LazyImport("cohere", globals(), "milvus_model.reranker.cohere")
-cross_encoder = LazyImport("cross_encoder", globals(), "milvus_model.reranker.cross_encoder")
-jinaai = LazyImport("jinaai", globals(), "milvus_model.reranker.jinaai")
-voyageai = LazyImport("voyageai", globals(), "milvus_model.reranker.voyageai")
+bgegreranker = LazyImport("bgereranker", globals(), "pymilvus.model.reranker.bgereranker")
+cohere = LazyImport("cohere", globals(), "pymilvus.model.reranker.cohere")
+cross_encoder = LazyImport("cross_encoder", globals(), "pymilvus.model.reranker.cross_encoder")
+jinaai = LazyImport("jinaai", globals(), "pymilvus.model.reranker.jinaai")
+voyageai = LazyImport("voyageai", globals(), "pymilvus.model.reranker.voyageai")
 
 def BGERerankFunction(*args, **kwargs):
     return bgegreranker.BGERerankFunction(*args, **kwargs)

milvus_model/reranker/bgereranker.py → src/pymilvus/model/reranker/bgereranker.py

@@ -2,8 +2,8 @@
 
 import torch
 
-from milvus_model.base import BaseRerankFunction, RerankResult
-from milvus_model.utils import import_FlagEmbedding, import_transformers
+from pymilvus.model.base import BaseRerankFunction, RerankResult
+from pymilvus.model.utils import import_FlagEmbedding, import_transformers
 
 import_FlagEmbedding()
 import_transformers()

milvus_model/reranker/cohere.py → src/pymilvus/model/reranker/cohere.py

@@ -1,7 +1,7 @@
 from typing import List, Optional
 
-from milvus_model.base import BaseRerankFunction, RerankResult
-from milvus_model.utils import import_cohere
+from pymilvus.model.base import BaseRerankFunction, RerankResult
+from pymilvus.model.utils import import_cohere
 
 import_cohere()
 import cohere

milvus_model/reranker/cross_encoder.py → src/pymilvus/model/reranker/cross_encoder.py

@@ -1,7 +1,7 @@
 from typing import Any, List
 
-from milvus_model.base import BaseRerankFunction, RerankResult
-from milvus_model.utils import import_sentence_transformers
+from pymilvus.model.base import BaseRerankFunction, RerankResult
+from pymilvus.model.utils import import_sentence_transformers
 
 import_sentence_transformers()
 import sentence_transformers

milvus_model/reranker/jinaai.py → src/pymilvus/model/reranker/jinaai.py

@@ -3,7 +3,7 @@
 
 import requests
 
-from milvus_model.base import BaseRerankFunction, RerankResult
+from pymilvus.model.base import BaseRerankFunction, RerankResult
 
 API_URL = "https://api.jina.ai/v1/rerank"
 

milvus_model/reranker/voyageai.py → src/pymilvus/model/reranker/voyageai.py

@@ -1,7 +1,7 @@
 from typing import List, Optional
 
-from milvus_model.base import BaseRerankFunction, RerankResult
-from milvus_model.utils import import_voyageai
+from pymilvus.model.base import BaseRerankFunction, RerankResult
+from pymilvus.model.utils import import_voyageai
 
 import_voyageai()
 import voyageai

milvus_model/sparse/__init__.py → src/pymilvus/model/sparse/__init__.py

@@ -1,10 +1,10 @@
 __all__ = ["SpladeEmbeddingFunction", "BM25EmbeddingFunction"]
 
 
-from milvus_model.utils.lazy_import import LazyImport
+from pymilvus.model.utils.lazy_import import LazyImport
 
-bm25 = LazyImport("bm25", globals(), "milvus_model.sparse.bm25")
-splade = LazyImport("openai", globals(), "milvus_model.sparse.splade")
+bm25 = LazyImport("bm25", globals(), "pymilvus.model.sparse.bm25")
+splade = LazyImport("openai", globals(), "pymilvus.model.sparse.splade")
 
 def BM25EmbeddingFunction(*args, **kwargs):
     return bm25.BM25EmbeddingFunction(*args, **kwargs)

milvus_model/sparse/bm25/__init__.py → src/pymilvus/model/sparse/bm25/__init__.py

@@ -5,10 +5,10 @@
     "build_default_analyzer",
 ]
 
-from milvus_model.utils.lazy_import import LazyImport
+from pymilvus.model.utils.lazy_import import LazyImport
 
-bm25 = LazyImport("bm25", globals(), "milvus_model.sparse.bm25.bm25")
-tokenizers = LazyImport("tokenizers", globals(), "milvus_model.sparse.bm25.tokenizers")
+bm25 = LazyImport("bm25", globals(), "pymilvus.model.sparse.bm25.bm25")
+tokenizers = LazyImport("tokenizers", globals(), "pymilvus.model.sparse.bm25.tokenizers")
 
 def BM25EmbeddingFunction(*args, **kwargs):
     return bm25.BM25EmbeddingFunction(*args, **kwargs)

milvus_model/sparse/bm25/bm25.py → src/pymilvus/model/sparse/bm25/bm25.py

@@ -26,8 +26,8 @@
 from scipy.sparse import csr_array, vstack
 import numpy as np
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.sparse.bm25.tokenizers import Analyzer, build_default_analyzer
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.sparse.bm25.tokenizers import Analyzer, build_default_analyzer
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)

milvus_model/sparse/bm25/tokenizers.py → src/pymilvus/model/sparse/bm25/tokenizers.py

@@ -4,7 +4,7 @@
 from importlib.util import find_spec
 from pathlib import Path
 from typing import Any, Dict, List, Match, Optional, Type
-from milvus_model.utils import import_nltk, import_jieba, import_mecab, import_konlpy, import_unidic_lite
+from pymilvus.model.utils import import_nltk, import_jieba, import_mecab, import_konlpy, import_unidic_lite
 
 import_nltk()
 import nltk

milvus_model/sparse/splade.py → src/pymilvus/model/sparse/splade.py

@@ -32,9 +32,9 @@
 import torch
 from scipy.sparse import csr_array
 
-from milvus_model.base import BaseEmbeddingFunction
-from milvus_model.utils import import_transformers, import_scipy, import_torch
-from milvus_model.sparse.utils import stack_sparse_embeddings
+from pymilvus.model.base import BaseEmbeddingFunction
+from pymilvus.model.utils import import_transformers, import_scipy, import_torch
+from pymilvus.model.sparse.utils import stack_sparse_embeddings
 
 import_torch()
 import_scipy()

milvus_model/utils/__init__.py → src/pymilvus/model/utils/__init__.py

@@ -23,7 +23,7 @@
 import importlib.util
 from typing import Optional
 
-from milvus_model.utils.dependency_control import prompt_install
+from pymilvus.model.utils.dependency_control import prompt_install
 
 def import_openai():
     _check_library("openai", package="openai>=1.12.0")