Implement RAG

.gitattributes

@@ -0,0 +1 @@
+assets/MiniLM-L6-H384-uncased/* filter=lfs diff=lfs merge=lfs -text

.licenserc.yaml

@@ -15,6 +15,7 @@ header:
     - test/**
   paths-ignore:
     - "**/BUILD"
+    - "**/*.json"
     - "**/*.bzl"
     - "**/*.proto"
     - lib/interop/generated_bindings.dart

lib/interop/generated_bindings.dart

@@ -151,6 +151,21 @@ class OpenVINO {
   late final _freeStatusOrDevices = _freeStatusOrDevicesPtr
       .asFunction<void Function(ffi.Pointer<StatusOrDevices>)>();
 
+  void freeStatusOrEmbeddings(
+    ffi.Pointer<StatusOrEmbeddings> status,
+  ) {
+    return _freeStatusOrEmbeddings(
+      status,
+    );
+  }
+
+  late final _freeStatusOrEmbeddingsPtr = _lookup<
+          ffi
+          .NativeFunction<ffi.Void Function(ffi.Pointer<StatusOrEmbeddings>)>>(
+      'freeStatusOrEmbeddings');
+  late final _freeStatusOrEmbeddings = _freeStatusOrEmbeddingsPtr
+      .asFunction<void Function(ffi.Pointer<StatusOrEmbeddings>)>();
+
   void freeStatusOrCameraDevices(
     ffi.Pointer<StatusOrCameraDevices> status,
   ) {
@@ -446,12 +461,14 @@ class OpenVINO {
   ffi.Pointer<StatusOrModelResponse> llmInferencePrompt(
     CLLMInference instance,
     ffi.Pointer<pkg_ffi.Utf8> message,
+    bool apply_template,
     double temperature,
     double top_p,
   ) {
     return _llmInferencePrompt(
       instance,
       message,
+      apply_template,
       temperature,
       top_p,
     );
@@ -462,11 +479,12 @@ class OpenVINO {
           ffi.Pointer<StatusOrModelResponse> Function(
               CLLMInference,
               ffi.Pointer<pkg_ffi.Utf8>,
+              ffi.Bool,
               ffi.Float,
               ffi.Float)>>('llmInferencePrompt');
   late final _llmInferencePrompt = _llmInferencePromptPtr.asFunction<
       ffi.Pointer<StatusOrModelResponse> Function(
-          CLLMInference, ffi.Pointer<pkg_ffi.Utf8>, double, double)>();
+          CLLMInference, ffi.Pointer<pkg_ffi.Utf8>, bool, double, double)>();
 
   ffi.Pointer<Status> llmInferenceClearHistory(
     CLLMInference instance,
@@ -496,20 +514,21 @@ class OpenVINO {
   late final _llmInferenceForceStop = _llmInferenceForceStopPtr
       .asFunction<ffi.Pointer<Status> Function(CLLMInference)>();
 
-  ffi.Pointer<StatusOrBool> llmInferenceHasChatTemplate(
+  ffi.Pointer<StatusOrString> llmInferenceGetTokenizerConfig(
     CLLMInference instance,
   ) {
-    return _llmInferenceHasChatTemplate(
+    return _llmInferenceGetTokenizerConfig(
       instance,
     );
   }
 
-  late final _llmInferenceHasChatTemplatePtr = _lookup<
+  late final _llmInferenceGetTokenizerConfigPtr = _lookup<
           ffi
-          .NativeFunction<ffi.Pointer<StatusOrBool> Function(CLLMInference)>>(
-      'llmInferenceHasChatTemplate');
-  late final _llmInferenceHasChatTemplate = _llmInferenceHasChatTemplatePtr
-      .asFunction<ffi.Pointer<StatusOrBool> Function(CLLMInference)>();
+          .NativeFunction<ffi.Pointer<StatusOrString> Function(CLLMInference)>>(
+      'llmInferenceGetTokenizerConfig');
+  late final _llmInferenceGetTokenizerConfig =
+      _llmInferenceGetTokenizerConfigPtr
+          .asFunction<ffi.Pointer<StatusOrString> Function(CLLMInference)>();
 
   ffi.Pointer<Status> llmInferenceClose(
     CLLMInference instance,
@@ -885,6 +904,74 @@ class OpenVINO {
   late final _graphRunnerStopCamera = _graphRunnerStopCameraPtr
       .asFunction<ffi.Pointer<Status> Function(CGraphRunner)>();
 
+  ffi.Pointer<StatusOrSentenceTransformer> sentenceTransformerOpen(
+    ffi.Pointer<pkg_ffi.Utf8> model_path,
+    ffi.Pointer<pkg_ffi.Utf8> device,
+  ) {
+    return _sentenceTransformerOpen(
+      model_path,
+      device,
+    );
+  }
+
+  late final _sentenceTransformerOpenPtr = _lookup<
+      ffi.NativeFunction<
+          ffi.Pointer<StatusOrSentenceTransformer> Function(
+              ffi.Pointer<pkg_ffi.Utf8>,
+              ffi.Pointer<pkg_ffi.Utf8>)>>('sentenceTransformerOpen');
+  late final _sentenceTransformerOpen = _sentenceTransformerOpenPtr.asFunction<
+      ffi.Pointer<StatusOrSentenceTransformer> Function(
+          ffi.Pointer<pkg_ffi.Utf8>, ffi.Pointer<pkg_ffi.Utf8>)>();
+
+  ffi.Pointer<StatusOrEmbeddings> sentenceTransformerGenerate(
+    CSentenceTransformer instance,
+    ffi.Pointer<pkg_ffi.Utf8> prompt,
+  ) {
+    return _sentenceTransformerGenerate(
+      instance,
+      prompt,
+    );
+  }
+
+  late final _sentenceTransformerGeneratePtr = _lookup<
+      ffi.NativeFunction<
+          ffi.Pointer<StatusOrEmbeddings> Function(CSentenceTransformer,
+              ffi.Pointer<pkg_ffi.Utf8>)>>('sentenceTransformerGenerate');
+  late final _sentenceTransformerGenerate =
+      _sentenceTransformerGeneratePtr.asFunction<
+          ffi.Pointer<StatusOrEmbeddings> Function(
+              CSentenceTransformer, ffi.Pointer<pkg_ffi.Utf8>)>();
+
+  ffi.Pointer<Status> sentenceTransformerClose(
+    CSentenceTransformer instance,
+  ) {
+    return _sentenceTransformerClose(
+      instance,
+    );
+  }
+
+  late final _sentenceTransformerClosePtr = _lookup<
+          ffi
+          .NativeFunction<ffi.Pointer<Status> Function(CSentenceTransformer)>>(
+      'sentenceTransformerClose');
+  late final _sentenceTransformerClose = _sentenceTransformerClosePtr
+      .asFunction<ffi.Pointer<Status> Function(CSentenceTransformer)>();
+
+  ffi.Pointer<StatusOrString> pdfExtractText(
+    ffi.Pointer<pkg_ffi.Utf8> pdf_path,
+  ) {
+    return _pdfExtractText(
+      pdf_path,
+    );
+  }
+
+  late final _pdfExtractTextPtr = _lookup<
+      ffi.NativeFunction<
+          ffi.Pointer<StatusOrString> Function(
+              ffi.Pointer<pkg_ffi.Utf8>)>>('pdfExtractText');
+  late final _pdfExtractText = _pdfExtractTextPtr.asFunction<
+      ffi.Pointer<StatusOrString> Function(ffi.Pointer<pkg_ffi.Utf8>)>();
+
   ffi.Pointer<StatusOrSpeechToText> speechToTextOpen(
     ffi.Pointer<pkg_ffi.Utf8> model_path,
     ffi.Pointer<pkg_ffi.Utf8> device,
@@ -1199,6 +1286,17 @@ final class StatusOrGraphRunner extends ffi.Struct {
 
 typedef CGraphRunner = ffi.Pointer<ffi.Void>;
 
+final class StatusOrSentenceTransformer extends ffi.Struct {
+  @ffi.Int()
+  external int status;
+
+  external ffi.Pointer<pkg_ffi.Utf8> message;
+
+  external CSentenceTransformer value;
+}
+
+typedef CSentenceTransformer = ffi.Pointer<ffi.Void>;
+
 final class StatusOrSpeechToText extends ffi.Struct {
   @ffi.Int()
   external int status;
@@ -1277,6 +1375,18 @@ final class StatusOrTTIModelResponse extends ffi.Struct {
   external ffi.Pointer<pkg_ffi.Utf8> value;
 }
 
+final class StatusOrEmbeddings extends ffi.Struct {
+  @ffi.Int()
+  external int status;
+
+  external ffi.Pointer<pkg_ffi.Utf8> message;
+
+  external ffi.Pointer<ffi.Float> value;
+
+  @ffi.Int()
+  external int size;
+}
+
 final class StatusOrVLMModelResponse extends ffi.Struct {
   @ffi.Int()
   external int status;

lib/interop/llm_inference.dart

@@ -13,11 +13,8 @@ class LLMInference {
 
   NativeCallable<LLMInferenceCallbackFunctionFunction>? nativeListener;
   final Pointer<StatusOrLLMInference> instance;
-  late bool chatEnabled;
 
-  LLMInference(this.instance) {
-    chatEnabled = hasChatTemplate();
-  }
+  LLMInference(this.instance);
 
   static Future<LLMInference> init(String modelPath, String device) async {
     final result = await Isolate.run(() {
@@ -38,11 +35,12 @@ class LLMInference {
     return LLMInference(result);
   }
 
-  Future<ModelResponse> prompt(String message, double temperature, double topP) async {
+  Future<ModelResponse> prompt(String message, bool applyTemplate, double temperature, double topP) async {
+    print("Actual prompt: $message");
     int instanceAddress = instance.ref.value.address;
     final result = await Isolate.run(() {
       final messagePtr = message.toNativeUtf8();
-      final status = llmOV.llmInferencePrompt(Pointer<Void>.fromAddress(instanceAddress), messagePtr, temperature, topP);
+      final status = llmOV.llmInferencePrompt(Pointer<Void>.fromAddress(instanceAddress), messagePtr, applyTemplate, temperature, topP);
       calloc.free(messagePtr);
       return status;
     })
@@ -83,14 +81,16 @@ class LLMInference {
     }
   }
 
-  bool hasChatTemplate() {
-    final status = llmOV.llmInferenceHasChatTemplate(instance.ref.value);
+  String getTokenizerConfig() {
+    final status = llmOV.llmInferenceGetTokenizerConfig(instance.ref.value);
 
     if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {
-      throw "LLM Chat template error: ${status.ref.status} ${status.ref.message.toDartString()}";
+      throw "LLM get Chat template error: ${status.ref.status} ${status.ref.message.toDartString()}";
     }
 
-    return status.ref.value;
+    final result = status.ref.value.toDartString();
+    llmOV.freeStatusOrString(status);
+    return result;
   }
 
   void close() {

lib/interop/pdf_extractor.dart

@@ -0,0 +1,31 @@
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+
+//EXPORT StatusOrSentences* pdfExtractSentences(const char* pdf_path);
+
+import 'dart:isolate';
+
+import 'dart:ffi';
+import 'package:ffi/ffi.dart';
+import 'package:inference/interop/openvino_bindings.dart';
+
+final ov = getBindings();
+
+
+Future<String> getTextFromPdf(String path) {
+  return Isolate.run(() {
+    final pathPtr = path.toNativeUtf8();
+    final status = ov.pdfExtractText(pathPtr);
+    calloc.free(pathPtr);
+
+    if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {
+      throw "pdfExtractText error: ${status.ref.status} ${status.ref.message.toDartString()}";
+    }
+
+    final output = status.ref.value.toDartString();
+    ov.freeStatusOrString(status);
+    return output;
+  });
+}

lib/interop/sentence_transformer.dart

@@ -0,0 +1,89 @@
+// Copyright (c) 2024 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+import 'dart:ffi';
+import 'dart:isolate';
+import 'dart:math';
+
+import 'package:ffi/ffi.dart';
+import 'package:inference/interop/openvino_bindings.dart';
+
+final ov = getBindings();
+
+class SentenceTransformer {
+
+  final Pointer<StatusOrSentenceTransformer> instance;
+  SentenceTransformer(this.instance);
+
+  static Future<SentenceTransformer> init(String modelPath, String device) async {
+    final result = await Isolate.run(() {
+      final modelPathPtr = modelPath.toNativeUtf8();
+      final devicePtr = device.toNativeUtf8();
+      final status = ov.sentenceTransformerOpen(modelPathPtr, devicePtr);
+      calloc.free(modelPathPtr);
+      calloc.free(devicePtr);
+      return status;
+    });
+
+
+    if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) {
+      throw "SentenceTransformer open error: ${result.ref.status} ${result.ref.message.toDartString()}";
+    }
+    return SentenceTransformer(result);
+  }
+
+  Future<List<double>> generate(String prompt) async{
+
+    int instanceAddress = instance.ref.value.address;
+    final status = await Isolate.run(() {
+      final promptPtr = prompt.toNativeUtf8();
+      final status = ov.sentenceTransformerGenerate(Pointer<Void>.fromAddress(instanceAddress), promptPtr);
+      calloc.free(promptPtr);
+      return status;
+    });
+
+    if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {
+      throw "SentenceTransformer generate error: ${status.ref.status} ${status.ref.message.toDartString()}";
+    }
+
+    List<double> data = [];
+    for (int i = 0; i < status.ref.size; i++) {
+      data.add(status.ref.value[i]);
+    }
+
+    ov.freeStatusOrEmbeddings(status);
+
+    return data;
+  }
+
+  void close() {
+    final status = ov.sentenceTransformerClose(instance.ref.value);
+    if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {
+      throw "Close error: ${status.ref.status} ${status.ref.message.toDartString()}";
+    }
+    ov.freeStatus(status);
+  }
+
+  static double cosineSimilarity(List<double> vec1, List<double> vec2) {
+    if (vec1.length != vec2.length) {
+        throw Exception("Vectors must be of the same size");
+    }
+
+    double dotProduct = 0.0;
+    double normVec1 = 0.0;
+    double normVec2 = 0.0;
+
+    for (int i = 0; i < vec1.length; ++i) {
+        dotProduct += vec1[i] * vec2[i];
+        normVec1 += vec1[i] * vec1[i];
+        normVec2 += vec2[i] * vec2[i];
+    }
+
+    if (normVec1 == 0 || normVec2 == 0) {
+        throw Exception("Vectors must not be zero-vectors");
+    }
+
+    return dotProduct / (sqrt(normVec1) * sqrt(normVec2));
+  }
+}