refactor: clean up bindings dir and move tests around

docs/developers/01_getting_started.md

@@ -22,9 +22,9 @@
     - [`packages/storage`](#packagesstorage)
     - [`packages/ai`](#packagesai)
     - [`packages/ai-provider`](#packagesai-provider)
-    - [`examples/cli`](#samplescli)
-    - [`examples/web`](#samplesweb)
-    - [`examples/ngraph`](#samplesngraph)
+    - [`examples/cli`](#examplescli)
+    - [`examples/web`](#examplesweb)
+    - [`examples/ngraph`](#examplesngraph)
 
 # Developer Getting Started
 
@@ -51,7 +51,7 @@ After this, plese read [Architecture](02_architecture.md) before attempting to [
 
 ```ts
 import { TaskGraphBuilder } from "ellmers-core";
-import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/server";
+import { registerHuggingfaceLocalTasksInMemory } from "ellmers-test";
 // config and start up
 registerHuggingfaceLocalTasksInMemory();
 
@@ -79,8 +79,8 @@ import {
   DataFlow,
   TaskGraph,
   TaskGraphRunner,
-  registerHuggingfaceLocalTasksInMemory,
 } from "ellmers-core";
+import { registerHuggingfaceLocalTasksInMemory } from "ellmers-test";
 
 // config and start up
 registerHuggingfaceLocalTasksInMemory();

examples/cli/src/ellmers.ts

@@ -4,8 +4,11 @@ import { program } from "commander";
 import { argv } from "process";
 import { AddBaseCommands } from "./TaskCLI";
 import { getProviderRegistry } from "ellmers-ai";
-import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/server";
-import { registerMediaPipeTfJsLocalInMemory } from "ellmers-ai-provider/tf-mediapipe/server";
+import {
+  registerHuggingfaceLocalTasksInMemory,
+  registerMediaPipeTfJsLocalInMemory,
+} from "ellmers-test";
+import "ellmers-test";
 
 program.version("1.0.0").description("A CLI to run Ellmers.");
 

examples/web/src/App.tsx

@@ -2,19 +2,47 @@ import React, { useCallback, useEffect, useState } from "react";
 import { ReactFlowProvider } from "@xyflow/react";
 import { RunGraphFlow } from "./RunGraphFlow";
 import { JsonEditor } from "./JsonEditor";
-import { JsonTask, JsonTaskItem, TaskGraph, TaskGraphBuilder } from "ellmers-core";
 import {
+  ConcurrencyLimiter,
+  JsonTask,
+  JsonTaskItem,
+  TaskGraph,
+  TaskGraphBuilder,
+  TaskInput,
+  TaskOutput,
+} from "ellmers-core";
+import {
+  IndexedDbQueue,
   IndexedDbTaskGraphRepository,
   IndexedDbTaskOutputRepository,
 } from "ellmers-storage/browser/indexeddb";
 import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from "./Resize";
 import { QueuesStatus } from "./QueueSatus";
 import { OutputRepositoryStatus } from "./OutputRepositoryStatus";
 import { GraphStoreStatus } from "./GraphStoreStatus";
-import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/browser";
+import { InMemoryJobQueue } from "ellmers-storage/inmemory";
+import { registerHuggingfaceLocalTasks } from "ellmers-ai-provider/hf-transformers/browser";
+import { getProviderRegistry, ModelProcessorEnum } from "ellmers-ai";
+import { registerMediaPipeTfJsLocalTasks } from "ellmers-ai-provider/tf-mediapipe/browser";
 import "ellmers-task";
+import "ellmers-test";
+
+const ProviderRegistry = getProviderRegistry();
+
+registerHuggingfaceLocalTasks();
+ProviderRegistry.registerQueue(
+  ModelProcessorEnum.LOCAL_ONNX_TRANSFORMERJS,
+  new InMemoryJobQueue<TaskInput, TaskOutput>("local_hft", new ConcurrencyLimiter(1, 10), 10)
+);
+
+registerMediaPipeTfJsLocalTasks();
+ProviderRegistry.registerQueue(
+  ModelProcessorEnum.MEDIA_PIPE_TFJS_MODEL,
+  new InMemoryJobQueue<TaskInput, TaskOutput>("local_mp", new ConcurrencyLimiter(1, 10), 10)
+);
 
-registerHuggingfaceLocalTasksInMemory();
+ProviderRegistry.clearQueues();
+ProviderRegistry.startQueues();
 
 const taskOutputCache = new IndexedDbTaskOutputRepository();
 const builder = new TaskGraphBuilder(taskOutputCache);

examples/web/src/RunGraphFlow.tsx

@@ -14,16 +14,11 @@ import { TurboNodeData, SingleNode, CompoundNode } from "./TurboNode";
 import TurboEdge from "./TurboEdge";
 import { FiFileText, FiClipboard, FiDownload, FiUpload } from "react-icons/fi";
 import { Task, TaskGraph } from "ellmers-core";
-import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/browser";
-import { registerMediaPipeTfJsLocalInMemory } from "ellmers-ai-provider/tf-mediapipe/browser";
 import { GraphPipelineCenteredLayout, GraphPipelineLayout, computeLayout } from "./layout";
 
 import "@xyflow/react/dist/base.css";
 import "./RunGraphFlow.css";
 
-registerHuggingfaceLocalTasksInMemory();
-registerMediaPipeTfJsLocalInMemory();
-
 const categoryIcons = {
   "Text Model": <FiFileText />,
   Input: <FiUpload />,

package.json

@@ -15,11 +15,12 @@
     "build:ai-provider": "cd packages/ai-provider && bun run build",
     "build:storage": "cd packages/storage && bun run build",
     "build:task": "cd packages/task && bun run build",
+    "build:test": "cd packages/test && bun run build",
     "build:examples": "bun run bun run build:cli && bun run build:web",
     "build:cli": "cd examples/cli && bun run build",
     "build:web": "cd examples/web && bun run build",
     "clean": "rm -rf node_modules packages/*/node_modules packages/*/dist packages/*/src/**/*\\.d\\.ts packages/*/src/**/*\\.map examples/*/node_modules examples/*/dist examples/*/src/**/*\\.d\\.ts examples/*/src/**/*\\.map",
-    "watch:packages": "concurrently --kill-others -c 'auto' -n core,task,storage,ai,provider 'cd packages/core && bun run watch' 'sleep 3 && cd packages/task && bun run watch' 'sleep 3 && cd packages/storage && bun run watch' 'sleep 3 && cd packages/ai && bun run watch' 'sleep 6 && cd packages/ai-provider && bun run watch'",
+    "watch:packages": "concurrently --kill-others -c 'auto' -n core,task,storage,ai,provider,test 'cd packages/core && bun run watch' 'sleep 3 && cd packages/task && bun run watch' 'sleep 3 && cd packages/storage && bun run watch' 'sleep 3 && cd packages/ai && bun run watch' 'sleep 6 && cd packages/ai-provider && bun run watch' 'sleep 10 && cd packages/test && bun run watch'",
     "docs": "typedoc",
     "format": "eslint \"{packages|examples}/*/src/**/*.{js,ts,tsx,json}\" --fix && prettier \"{packages|examples}/*/src/**/*.{js,ts,tsx,json}\" --check --write",
     "release": "bun run build && bun publish",

packages/ai-provider/src/hf-transformers/browser.ts

@@ -7,5 +7,4 @@
 
 export * from "./provider/HuggingFaceLocal_TaskRun";
 export * from "./model/ONNXTransformerJsModel";
-export * from "./bindings/local_hf";
-export * from "./bindings/all_inmemory";
+export * from "./bindings/registerTasks";

packages/ai-provider/src/hf-transformers/test/HFTransformersBinding.test.ts

@@ -0,0 +1,79 @@
+//    *******************************************************************************
+//    *   ELLMERS: Embedding Large Language Model Experiential Retrieval Service    *
+//    *                                                                             *
+//    *   Copyright Steven Roussey <[email protected]>                             *
+//    *   Licensed under the Apache License, Version 2.0 (the "License");           *
+//    *******************************************************************************
+
+import { describe, expect, it } from "bun:test";
+import { ConcurrencyLimiter, TaskGraphBuilder, TaskInput, TaskOutput } from "ellmers-core";
+import { getProviderRegistry, ModelProcessorEnum, ModelUseCaseEnum } from "ellmers-ai";
+import { InMemoryJobQueue } from "ellmers-storage/inmemory";
+import { SqliteJobQueue } from "ellmers-storage/bun/sqlite";
+import { registerHuggingfaceLocalTasks } from "../bindings/registerTasks";
+import { getDatabase } from "../../../../storage/src/util/db_sqlite";
+import { sleep } from "bun";
+import { ONNXTransformerJsModel } from "../model/ONNXTransformerJsModel";
+
+const HFQUEUE = "local_hf";
+
+describe("HFTransformersBinding", () => {
+  describe("InMemoryJobQueue", () => {
+    it("Should have an item queued", async () => {
+      // the model gets self-registered
+      const flanT5p786m = new ONNXTransformerJsModel(
+        "Xenova/LaMini-Flan-T5-783M",
+        [ModelUseCaseEnum.TEXT_GENERATION, ModelUseCaseEnum.TEXT_REWRITING],
+        "text2text-generation"
+      );
+      registerHuggingfaceLocalTasks();
+      const providerRegistry = getProviderRegistry();
+      const jobQueue = new InMemoryJobQueue<TaskInput, TaskOutput>(
+        HFQUEUE,
+        new ConcurrencyLimiter(1, 10),
+        10
+      );
+      providerRegistry.registerQueue(ModelProcessorEnum.LOCAL_ONNX_TRANSFORMERJS, jobQueue);
+      const queue = providerRegistry.getQueue(ModelProcessorEnum.LOCAL_ONNX_TRANSFORMERJS);
+      expect(queue).toBeDefined();
+      expect(queue?.queue).toEqual(HFQUEUE);
+
+      const builder = new TaskGraphBuilder();
+      builder.DownloadModel({
+        model: "Xenova/LaMini-Flan-T5-783M",
+      });
+      builder.run();
+      await sleep(1);
+      expect(await queue?.size()).toEqual(1);
+      await queue?.clear();
+    });
+  });
+
+  describe("SqliteJobQueue", () => {
+    it("Should have an item queued", async () => {
+      registerHuggingfaceLocalTasks();
+      const providerRegistry = getProviderRegistry();
+      const jobQueue = new SqliteJobQueue<TaskInput, TaskOutput>(
+        getDatabase(),
+        HFQUEUE,
+        new ConcurrencyLimiter(1, 10),
+        10
+      );
+      jobQueue.ensureTableExists();
+      providerRegistry.registerQueue(ModelProcessorEnum.LOCAL_ONNX_TRANSFORMERJS, jobQueue);
+      const queue = providerRegistry.getQueue(ModelProcessorEnum.LOCAL_ONNX_TRANSFORMERJS);
+      expect(queue).toBeDefined();
+      expect(queue?.queue).toEqual(HFQUEUE);
+
+      const builder = new TaskGraphBuilder();
+      builder.DownloadModel({
+        model: "Xenova/LaMini-Flan-T5-783M",
+      });
+      builder.run();
+      await sleep(1);
+      expect(await queue?.size()).toEqual(1);
+      builder.reset();
+      await queue?.clear();
+    });
+  });
+});

packages/ai-provider/src/tf-mediapipe/browser.ts

@@ -7,5 +7,4 @@
 
 export * from "./provider/MediaPipeLocalTaskRun";
 export * from "./model/MediaPipeModel";
-export * from "./bindings/local_mp";
-export * from "./bindings/all_inmemory";
+export * from "./bindings/registerTasks";