Merge pull request #8 from sroussey/reorg

refactor: reorganize for future enhancements

.vscode/settings.json

@@ -10,7 +10,9 @@
   "files.exclude": {
     // "**/dist": true,
     // "**/build": true,
-    "**/*.tsbuildinfo": true
+    "**/*.tsbuildinfo": true,
+    "**/node_modules": true,
+    "**/dist": true
   },
   "search.exclude": {
     "**/dist": true,

README.md

@@ -25,7 +25,7 @@ The Embedding Large Language Model Experiential retrieval Service (ELLMERS) is a
 - **[Architecture](docs/developers/02_architecture.md)**
 - **[Extending the System](docs/developers/03_extending.md)**
 
-## Examples
+## Samples
 
 ### CLI
 

docs/developers/01_getting_started.md

@@ -19,9 +19,12 @@
   - [Source](#source)
     - [`docs/`](#docs)
     - [`packages/core`](#packagescore)
-    - [`packages/cli`](#packagescli)
-    - [`packages/web`](#packagesweb)
-    - [`packages/ngraph`](#packagesngraph)
+    - [`packages/storage`](#packagesstorage)
+    - [`packages/ai`](#packagesai)
+    - [`packages/ai-provider`](#packagesai-provider)
+    - [`examples/cli`](#samplescli)
+    - [`examples/web`](#samplesweb)
+    - [`examples/ngraph`](#samplesngraph)
 
 # Developer Getting Started
 
@@ -32,7 +35,7 @@ git clone https://github.com/sroussey/ellmers.git
 cd ellmers
 bun install
 bun run build
-cd packages/web
+cd examples/web
 bun run dev
 ```
 
@@ -47,7 +50,8 @@ After this, plese read [Architecture](02_architecture.md) before attempting to [
 ## Using TaskGraphBuilder & a config helper
 
 ```ts
-import { TaskGraphBuilder, registerHuggingfaceLocalTasksInMemory } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
+import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/server";
 // config and start up
 registerHuggingfaceLocalTasksInMemory();
 
@@ -76,7 +80,7 @@ import {
   TaskGraph,
   TaskGraphRunner,
   registerHuggingfaceLocalTasksInMemory,
-} from "ellmers-core/server";
+} from "ellmers-core";
 
 // config and start up
 registerHuggingfaceLocalTasksInMemory();
@@ -135,7 +139,7 @@ import {
   ConcurrencyLimiter,
   TaskInput,
   TaskOutput,
-} from "ellmers-core/server";
+} from "ellmers-core";
 
 // config and start up
 const ProviderRegistry = getProviderRegistry();
@@ -230,7 +234,7 @@ Tasks are the smallest unit of work, therefore they take simple inputs. Most Tas
 An example is TextEmbeddingTask and TextEmbeddingCompoundTask. The first takes a single model input, the second accepts an array of model inputs. Since models can have different providers, the Compound version creates a single task version for each model input. The builder is smart enough to know that the Compound version is needed when an array is passed, and as such, you don't need to differentiate between the two:
 
 ```ts
-import { TaskGraphBuilder } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder.TextEmbedding({
   model: "Xenova/LaMini-Flan-T5-783M",
@@ -242,7 +246,7 @@ await builder.run();
 OR
 
 ```ts
-import { TaskGraphBuilder } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder.TextEmbedding({
   model: ["Xenova/LaMini-Flan-T5-783M", "Universal Sentence Encoder"],
@@ -254,7 +258,7 @@ await builder.run();
 The builder will look at outputs of one task and automatically connect it to the input of the next task, if the output and input names and types match. If they don't, you can use the `rename` method to rename the output of the first task to match the input of the second task.
 
 ```ts
-import { TaskGraphBuilder } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder
   .DownloadModel({
@@ -337,7 +341,7 @@ There is a JSONTask that can be used to build a graph. This is useful for saving
 The JSON above is a good example as it shows how to use a compound task with multiple inputs. Compound tasks export arrays, so use a compound task to consume the output of another compound task. The `dependencies` object is used to specify which output of which task is used as input for the current task. It is a shorthand for creating a data flow (an edge) in the graph.
 
 ```ts
-import { JSONTask } from "ellmers-core/server";
+import { JSONTask } from "ellmers-core";
 const json = require("./example.json");
 const task = new JSONTask({ input: { json } });
 await task.run();
@@ -438,21 +442,33 @@ You are here.
 
 This is the main library code.
 
-### `packages/cli`
+### `packages/storage`
+
+Storage for queues, caches, etc.
+
+### `packages/ai`
+
+These are the LLM tasks, models, etc.
+
+### `packages/ai-provider`
+
+This is the Huggingface Transformers JS (using ONNX) and TensorFlow MediaPipe providers.
+
+### `examples/cli`
 
 An example project that uses the library in a CLI settings using listr2 (`cat example.json | ellmers json`, for example)
 
 ![cli example](img/cli.png)
 
-### `packages/web`
+### `examples/web`
 
 An example project that uses the library in a web setting, running locally in browser.
 
 ![web example](img/web.png)
 
 Don't forget to open the console for some goodies.
 
-### `packages/ngraph`
+### `examples/ngraph`
 
 A graph editor tool that uses ngraph. It is not yet ready for prime time.
 

packages/cli/package.json → examples/cli/package.json

@@ -5,11 +5,11 @@
   "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.",
   "scripts": {
     "watch": "concurrently -c 'auto' -n 'cli:' 'bun:watch-*'",
-    "watch-js": "bun build --watch --target=node --sourcemap=external --external listr2 --external @huggingface/transformers --outdir ./dist ./src/lib.ts ./src/ellmers.ts",
+    "watch-js": "bun build --watch --target=node --sourcemap=external --splitting --outdir ./dist ./src/lib.ts ./src/ellmers.ts",
     "watch-types": "tsc --watch --preserveWatchOutput",
     "build": "bun run build-clean && bun run build-types && bun run build-js",
     "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo",
-    "build-js": "bun build --target=node --sourcemap=external --external listr2 --external @huggingface/transformers --outdir ./dist ./src/lib.ts ./src/ellmers.ts",
+    "build-js": "bun build --target=node --sourcemap=external --splitting --outdir ./dist ./src/lib.ts ./src/ellmers.ts",
     "build-types": "tsc",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -24,6 +24,10 @@
     "dist"
   ],
   "dependencies": {
-    "ellmers-core": "workspace:packages/core"
+    "ellmers-core": "workspace:packages/core",
+    "ellmers-ai": "workspace:packages/ai",
+    "ellmers-storage": "workspace:packages/storage",
+    "ellmers-ai-provider": "workspace:packages/ai-provider",
+    "ellmers-task": "workspace:packages/task"
   }
 }

packages/cli/src/TaskCLI.ts → examples/cli/src/TaskCLI.ts

@@ -8,18 +8,17 @@
 import { Command } from "commander";
 import { runTask } from "./TaskStreamToListr2";
 import "@huggingface/transformers";
+import { TaskGraph, JsonTask, TaskGraphBuilder, JsonTaskItem } from "ellmers-core";
+
 import {
+  DownloadModelTask,
+  DownloadModelCompoundTask,
   findAllModels,
   findModelByName,
   findModelByUseCase,
-  DownloadModelTask,
   ModelUseCaseEnum,
-  TaskGraph,
-  JsonTask,
-  TaskGraphBuilder,
-  DownloadModelCompoundTask,
-  JsonTaskItem,
-} from "ellmers-core/server";
+} from "ellmers-ai";
+import "ellmers-task";
 
 export function AddBaseCommands(program: Command) {
   program

packages/cli/src/TaskStreamToListr2.ts → examples/cli/src/TaskStreamToListr2.ts

@@ -14,8 +14,7 @@ import {
   TaskGraphRunner,
   type Task,
   CompoundTask,
-  SqliteTaskOutputRepository,
-} from "ellmers-core/server";
+} from "ellmers-core";
 import { createBar } from "./TaskHelper";
 
 const options = {

packages/cli/src/ellmers.ts → examples/cli/src/ellmers.ts

@@ -3,11 +3,9 @@
 import { program } from "commander";
 import { argv } from "process";
 import { AddBaseCommands } from "./TaskCLI";
-import {
-  getProviderRegistry,
-  registerHuggingfaceLocalTasksInMemory,
-  registerMediaPipeTfJsLocalInMemory,
-} from "ellmers-core/server";
+import { getProviderRegistry } from "ellmers-ai";
+import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/server";
+import { registerMediaPipeTfJsLocalInMemory } from "ellmers-ai-provider/tf-mediapipe/server";
 
 program.version("1.0.0").description("A CLI to run Ellmers.");
 

examples/cli/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["src/**/*"],
+  "files": ["src/lib.ts", "src/ellmers.ts"],
+  "exclude": ["**/*.test.ts"],
+  "compilerOptions": {
+    "outDir": "dist",
+    "baseUrl": "./src",
+    "rootDir": "./src",
+    "paths": {
+      "#/*": ["./src/*"],
+      "ellmers-core": ["../../packages/core/src"],
+      "ellmers-ai": ["../../packages/ai/src"],
+      "ellmers-storage": ["../../packages/storage/src"],
+      "ellmers-ai-provider": ["../../packages/ai-provider/src"]
+    }
+  },
+  "references": [
+    { "path": "../../packages/core" },
+    { "path": "../../packages/ai" },
+    { "path": "../../packages/ai-provider" },
+    { "path": "../../packages/task" },
+    { "path": "../../packages/storage" }
+  ]
+}

packages/web/package.json → examples/web/package.json

@@ -21,7 +21,10 @@
     "react-hotkeys-hook": "^4.6.1",
     "react-icons": "^5.4.0",
     "react-resizable-panels": "^2.1.7",
-    "ellmers-core": "workspace:packages/core"
+    "ellmers-core": "workspace:packages/core",
+    "ellmers-storage": "workspace:packages/storage",
+    "ellmers-ai-provider": "workspace:packages/ai-provider",
+    "ellmers-ai": "workspace:packages/ai"
   },
   "devDependencies": {
     "@types/react": "^19.0.4",

packages/web/src/App.tsx → examples/web/src/App.tsx

@@ -2,18 +2,19 @@ import React, { useCallback, useEffect, useState } from "react";
 import { ReactFlowProvider } from "@xyflow/react";
 import { RunGraphFlow } from "./RunGraphFlow";
 import { JsonEditor } from "./JsonEditor";
+import { JsonTask, JsonTaskItem, TaskGraph, TaskGraphBuilder } from "ellmers-core";
 import {
   IndexedDbTaskGraphRepository,
   IndexedDbTaskOutputRepository,
-  JsonTask,
-  JsonTaskItem,
-  TaskGraph,
-  TaskGraphBuilder,
-} from "ellmers-core/browser";
+} from "ellmers-storage/browser/indexeddb";
 import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from "./Resize";
 import { QueuesStatus } from "./QueueSatus";
 import { OutputRepositoryStatus } from "./OutputRepositoryStatus";
 import { GraphStoreStatus } from "./GraphStoreStatus";
+import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/browser";
+import "ellmers-task";
+
+registerHuggingfaceLocalTasksInMemory();
 
 const taskOutputCache = new IndexedDbTaskOutputRepository();
 const builder = new TaskGraphBuilder(taskOutputCache);

packages/web/src/ConsoleFormatters.tsx → examples/web/src/ConsoleFormatters.tsx

@@ -3,7 +3,7 @@ import {
   TaskInputDefinition,
   TaskOutputDefinition,
   TaskStatus,
-} from "ellmers-core/browser";
+} from "ellmers-core";
 
 type Config = Record<string, any>;
 

packages/web/src/GraphStoreStatus.tsx → examples/web/src/GraphStoreStatus.tsx

@@ -1,4 +1,4 @@
-import { TaskGraphRepository } from "ellmers-core/browser";
+import { TaskGraphRepository } from "ellmers-core";
 import { useCallback } from "react";
 
 export function GraphStoreStatus({ repository }: { repository: TaskGraphRepository }) {

packages/web/src/JsonEditor.tsx → examples/web/src/JsonEditor.tsx

@@ -2,7 +2,7 @@ import React, { useState, useEffect } from "react";
 import CodeMirror from "@uiw/react-codemirror";
 import { vscodeDark } from "@uiw/codemirror-theme-vscode";
 import { json } from "@codemirror/lang-json";
-import { JsonTask } from "ellmers-core/browser";
+import { JsonTask } from "ellmers-core";
 
 import "./JsonEditor.css";
 

packages/web/src/OutputRepositoryStatus.tsx → examples/web/src/OutputRepositoryStatus.tsx

@@ -1,4 +1,4 @@
-import { TaskOutputRepository } from "ellmers-core/browser";
+import { TaskOutputRepository } from "ellmers-core";
 import { useCallback, useEffect, useState } from "react";
 
 export function OutputRepositoryStatus({ repository }: { repository: TaskOutputRepository }) {

packages/web/src/QueueSatus.tsx → examples/web/src/QueueSatus.tsx

@@ -1,4 +1,5 @@
-import { Job, JobStatus, ModelProcessorEnum, getProviderRegistry } from "ellmers-core/browser";
+import { JobStatus } from "ellmers-core";
+import { ModelProcessorEnum, getProviderRegistry } from "ellmers-ai";
 import { useCallback, useEffect, useState } from "react";
 
 export function QueueStatus({ queueType }: { queueType: ModelProcessorEnum }) {

packages/web/src/RunGraphFlow.tsx → examples/web/src/RunGraphFlow.tsx

@@ -1,4 +1,4 @@
-import React, { Dispatch, SetStateAction, useCallback, useEffect, useRef } from "react";
+import React, { Dispatch, SetStateAction, useEffect, useRef } from "react";
 import {
   ReactFlow,
   Controls,
@@ -13,12 +13,9 @@ import {
 import { TurboNodeData, SingleNode, CompoundNode } from "./TurboNode";
 import TurboEdge from "./TurboEdge";
 import { FiFileText, FiClipboard, FiDownload, FiUpload } from "react-icons/fi";
-import {
-  Task,
-  TaskGraph,
-  registerHuggingfaceLocalTasksInMemory,
-  registerMediaPipeTfJsLocalInMemory,
-} from "ellmers-core/browser";
+import { Task, TaskGraph } from "ellmers-core";
+import { registerHuggingfaceLocalTasksInMemory } from "ellmers-ai-provider/hf-transformers/browser";
+import { registerMediaPipeTfJsLocalInMemory } from "ellmers-ai-provider/tf-mediapipe/browser";
 import { GraphPipelineCenteredLayout, GraphPipelineLayout, computeLayout } from "./layout";
 
 import "@xyflow/react/dist/base.css";

packages/web/src/main.tsx → examples/web/src/main.tsx

@@ -1,6 +1,6 @@
 import ReactDOM from "react-dom/client";
 import { App } from "./App";
-import { TaskGraphBuilder } from "ellmers-core/browser";
+import { TaskGraphBuilder } from "ellmers-core";
 import "./main.css";
 import {
   TaskConsoleFormatter,

packages/web/tsconfig.json → examples/web/tsconfig.json

@@ -19,9 +19,18 @@
     "noFallthroughCasesInSwitch": true,
 
     "paths": {
-      "ellmers-core": ["./packages/core/"]
+      "ellmers-core": ["../../packages/core/src"],
+      "ellmers-ai-provider": ["../../packages/ai-provider/src"],
+      "ellmers-storage": ["../../packages/storage/src"],
+      "ellmers-task": ["../../packages/task/src"]
     }
   },
-  "include": ["src", "vite.config.js"],
-  "exclude": ["dist"]
+  "include": ["src"],
+  "exclude": ["dist"],
+  "references": [
+    { "path": "../../packages/core" },
+    { "path": "../../packages/task" },
+    { "path": "../../packages/ai-provider" },
+    { "path": "../../packages/storage" }
+  ]
 }