refactor: rework providers a bit and pull storage out of core

README.md

@@ -25,7 +25,7 @@ The Embedding Large Language Model Experiential retrieval Service (ELLMERS) is a
 - **[Architecture](docs/developers/02_architecture.md)**
 - **[Extending the System](docs/developers/03_extending.md)**
 
-## Examples
+## Samples
 
 ### CLI
 

docs/developers/01_getting_started.md

@@ -19,11 +19,12 @@
   - [Source](#source)
     - [`docs/`](#docs)
     - [`packages/core`](#packagescore)
-    - [`packages/provider-hf-transformers`](#packagesprovider-hf-transformers)
-    - [`packages/provider-tf-mediapipe`](#packagesprovider-tf-mediapipe)
-    - [`examples/cli`](#examplescli)
-    - [`examples/web`](#examplesweb)
-    - [`examples/ngraph`](#examplesngraph)
+    - [`packages/storage`](#packagesstorage)
+    - [`packages/task-llm`](#packagestask-llm)
+    - [`packages/provider`](#packagesprovider)
+    - [`samples/cli`](#samplescli)
+    - [`samples/web`](#samplesweb)
+    - [`samples/ngraph`](#samplesngraph)
 
 # Developer Getting Started
 
@@ -34,7 +35,7 @@ git clone https://github.com/sroussey/ellmers.git
 cd ellmers
 bun install
 bun run build
-cd examples/web
+cd samples/web
 bun run dev
 ```
 
@@ -49,7 +50,8 @@ After this, plese read [Architecture](02_architecture.md) before attempting to [
 ## Using TaskGraphBuilder & a config helper
 
 ```ts
-import { TaskGraphBuilder, registerHuggingfaceLocalTasksInMemory } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
+import { registerHuggingfaceLocalTasksInMemory } from "ellmers-provider/hf-transformers/server";
 // config and start up
 registerHuggingfaceLocalTasksInMemory();
 
@@ -78,7 +80,7 @@ import {
   TaskGraph,
   TaskGraphRunner,
   registerHuggingfaceLocalTasksInMemory,
-} from "ellmers-core/server";
+} from "ellmers-core";
 
 // config and start up
 registerHuggingfaceLocalTasksInMemory();
@@ -137,7 +139,7 @@ import {
   ConcurrencyLimiter,
   TaskInput,
   TaskOutput,
-} from "ellmers-core/server";
+} from "ellmers-core";
 
 // config and start up
 const ProviderRegistry = getProviderRegistry();
@@ -232,7 +234,7 @@ Tasks are the smallest unit of work, therefore they take simple inputs. Most Tas
 An example is TextEmbeddingTask and TextEmbeddingCompoundTask. The first takes a single model input, the second accepts an array of model inputs. Since models can have different providers, the Compound version creates a single task version for each model input. The builder is smart enough to know that the Compound version is needed when an array is passed, and as such, you don't need to differentiate between the two:
 
 ```ts
-import { TaskGraphBuilder } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder.TextEmbedding({
   model: "Xenova/LaMini-Flan-T5-783M",
@@ -244,7 +246,7 @@ await builder.run();
 OR
 
 ```ts
-import { TaskGraphBuilder } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder.TextEmbedding({
   model: ["Xenova/LaMini-Flan-T5-783M", "Universal Sentence Encoder"],
@@ -256,7 +258,7 @@ await builder.run();
 The builder will look at outputs of one task and automatically connect it to the input of the next task, if the output and input names and types match. If they don't, you can use the `rename` method to rename the output of the first task to match the input of the second task.
 
 ```ts
-import { TaskGraphBuilder } from "ellmers-core/server";
+import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder
   .DownloadModel({
@@ -339,7 +341,7 @@ There is a JSONTask that can be used to build a graph. This is useful for saving
 The JSON above is a good example as it shows how to use a compound task with multiple inputs. Compound tasks export arrays, so use a compound task to consume the output of another compound task. The `dependencies` object is used to specify which output of which task is used as input for the current task. It is a shorthand for creating a data flow (an edge) in the graph.
 
 ```ts
-import { JSONTask } from "ellmers-core/server";
+import { JSONTask } from "ellmers-core";
 const json = require("./example.json");
 const task = new JSONTask({ input: { json } });
 await task.run();
@@ -440,29 +442,33 @@ You are here.
 
 This is the main library code.
 
-### `packages/provider-hf-transformers`
+### `packages/storage`
 
-This is the Huggingface Transformers JS (using ONNX)provider.
+Storage for queues, caches, etc.
 
-### `packages/provider-tf-mediapipe`
+### `packages/task-llm`
 
-This is the TensorFlow MediaPipe provider.
+These are the LLM tasks.
 
-### `examples/cli`
+### `packages/provider`
+
+This is the Huggingface Transformers JS (using ONNX) and TensorFlow MediaPipe providers.
+
+### `samples/cli`
 
 An example project that uses the library in a CLI settings using listr2 (`cat example.json | ellmers json`, for example)
 
 ![cli example](img/cli.png)
 
-### `examples/web`
+### `samples/web`
 
 An example project that uses the library in a web setting, running locally in browser.
 
 ![web example](img/web.png)
 
 Don't forget to open the console for some goodies.
 
-### `examples/ngraph`
+### `samples/ngraph`
 
 A graph editor tool that uses ngraph. It is not yet ready for prime time.
 

package.json

@@ -8,18 +8,19 @@
     "./samples/*"
   ],
   "scripts": {
-    "build": "bun run build:packages && bun run build:examples",
-    "build:packages": "bun run build:core && bun run build:task-llm && bun run build:providers",
+    "build": "bun run build:packages && bun run build:samples",
+    "build:packages": "bun run build:core && bun run build:storage &&bun run build:task-llm && bun run build:provider",
     "build:core": "cd packages/core && bun run build",
     "build:task-llm": "cd packages/task-llm && bun run build",
-    "build:providers": "(cd packages/provider-hf-transformers && bun run build) && (cd packages/provider-tf-mediapipe && bun run build)",
-    "build:examples": "bun run bun run build:cli && bun run build:web",
+    "build:storage": "cd packages/storage && bun run build",
+    "build:provider": "cd packages/provider && bun run build",
+    "build:samples": "bun run bun run build:cli && bun run build:web",
     "build:cli": "cd samples/cli && bun run build",
     "build:web": "cd samples/web && bun run build",
-    "clean": "rm -rf node_modules packages/*/node_modules packages/*/dist packages/*/src/**/*\\.d\\.ts packages/*/src/**/*\\.map examples/*/node_modules examples/*/dist examples/*/src/**/*\\.d\\.ts examples/*/src/**/*\\.map",
-    "watch:packages": "concurrently --kill-others -c 'auto' -n core,task-llm,hf-trans,tf-media 'cd packages/core && bun run watch' 'cd packages/task-llm && bun run watch' 'sleep 5 && cd packages/provider-hf-transformers && bun run watch' 'sleep 5 && cd packages/provider-tf-mediapipe && bun run watch'",
+    "clean": "rm -rf node_modules packages/*/node_modules packages/*/dist packages/*/src/**/*\\.d\\.ts packages/*/src/**/*\\.map samples/*/node_modules samples/*/dist samples/*/src/**/*\\.d\\.ts samples/*/src/**/*\\.map",
+    "watch:packages": "concurrently --kill-others -c 'auto' -n core,storage,task-llm,provider 'cd packages/core && bun run watch' 'sleep 3 && cd packages/storage && bun run watch' 'sleep 3 && cd packages/task-llm && bun run watch' 'sleep 6 && cd packages/provider && bun run watch'",
     "docs": "typedoc",
-    "format": "eslint \"{packages|examples}/*/src/**/*.{js,ts,tsx,json}\" --fix && prettier \"{packages|examples}/*/src/**/*.{js,ts,tsx,json}\" --check --write",
+    "format": "eslint \"{packages|samples}/*/src/**/*.{js,ts,tsx,json}\" --fix && prettier \"{packages|samples}/*/src/**/*.{js,ts,tsx,json}\" --check --write",
     "release": "bun run build && bun publish",
     "test": "jest"
   },

packages/core/package.json

@@ -4,14 +4,12 @@
   "version": "0.0.1",
   "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.",
   "scripts": {
-    "watch": "concurrently -c 'auto' -n 'core:' 'bun:watch-*'",
-    "watch-browser": "bun build --watch --target=browser --sourcemap=external --external @huggingface/transformers --outdir ./dist ./src/index.ts",
-    "watch-server-bun": "bun build --watch --target=bun --sourcemap=external --external @huggingface/transformers --outdir ./dist ./src/server.ts",
+    "watch": "concurrently -c 'auto' 'bun:watch-*'",
+    "watch-code": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external @huggingface/transformers --outdir ./dist ./src/index.ts",
     "watch-types": "tsc --watch --preserveWatchOutput",
-    "build": "bun run build-clean && bun run build-types && bun run build-browser && bun run build-server-bun",
+    "build": "bun run build-clean && bun run build-types && bun run build-code",
     "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo",
-    "build-browser": "bun build --target=browser --sourcemap=external --external @huggingface/transformers --outdir ./dist ./src/index.ts",
-    "build-server-bun": "bun build --target=bun --sourcemap=external --external @huggingface/transformers --outdir ./dist ./src/server.ts",
+    "build-code": "bun build --target=browser --sourcemap=external --external @huggingface/transformers --outdir ./dist ./src/index.ts",
     "build-types": "tsc",
     "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
     "test": "bun test"
@@ -22,14 +20,6 @@
     ".": {
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
-    },
-    "./browser": {
-      "import": "./dist/index.js",
-      "types": "./dist/index.d.ts"
-    },
-    "./server": {
-      "import": "./dist/server.js",
-      "types": "./dist/server.d.ts"
     }
   },
   "files": [

packages/core/src/index.ts

@@ -12,12 +12,11 @@ export * from "./task/index";
 export * from "./provider/ProviderRegistry";
 export * from "./model/Model";
 export * from "./model/InMemoryStorage";
+export * from "./storage/base/KVRepository";
 export * from "./storage/taskoutput/TaskOutputRepository";
 export * from "./storage/taskoutput/InMemoryTaskOutputRepository";
-export * from "./storage/taskoutput/IndexedDbTaskOutputRepository";
 export * from "./storage/taskgraph/TaskGraphRepository";
 export * from "./storage/taskgraph/InMemoryTaskGraphRepository";
-export * from "./storage/taskgraph/IndexedDbTaskGraphRepository";
 export * from "./util/Misc";
 export * from "./job/base/Job";
 export * from "./job/base/JobQueue";

packages/core/src/storage/base/SqliteKVRepository.ts

@@ -15,7 +15,7 @@ import { makeFingerprint } from "../../util/Misc";
 export class SqliteKVRepository<
   Key = string,
   Value = string,
-  Discriminator extends DiscriminatorSchema = DiscriminatorSchema,
+  Discriminator extends DiscriminatorSchema = DiscriminatorSchema
 > extends KVRepository<Key, Value, Discriminator> {
   private db: Database;
   constructor(

packages/core/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "../../tsconfig.json",
   "include": ["src/**/*"],
-  "files": ["src/index.ts", "src/server.ts"],
+  "files": ["src/index.ts"],
   "exclude": ["**/*.test.ts", "dist"],
   "compilerOptions": {
     "outDir": "./dist",

packages/provider/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "ellmers-provider",
+  "type": "module",
+  "version": "0.0.1",
+  "description": "Ellmers is a tool for building and running DAG pipelines of AI tasks.",
+  "scripts": {
+    "watch": "concurrently -c 'auto' 'bun:watch-*'",
+    "watch-hf-transformers-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external @huggingface/transformers --external ellmers-core --outdir ./dist/hf-transformers ./src/hf-transformers/browser.ts",
+    "watch-hf-transformers-server": "bun build --watch --no-clear-screen --target=node --sourcemap=external --external @huggingface/transformers --external ellmers-core --outdir ./dist/hf-transformers ./src/hf-transformers/server.ts",
+    "watch-hf-transformers-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external @huggingface/transformers --external ellmers-core --outdir ./dist/hf-transformers ./src/hf-transformers/worker.ts",
+    "watch-tf-mediapipe-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/browser.ts",
+    "watch-tf-mediapipe-server": "bun build --watch --no-clear-screen --target=node --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/server.ts",
+    "watch-tf-mediapipe-worker": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/worker.ts",
+    "watch-types": "tsc --watch --preserveWatchOutput",
+    "build": "bun run build-clean && bun run build-types && bun run build-hf-transformers && bun run build-tf-mediapipe",
+    "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo",
+    "build-hf-transformers": "bun run build-hf-transformers-browser && bun run build-hf-transformers-server && bun run build-hf-transformers-worker",
+    "build-hf-transformers-browser": "bun build --target=browser --sourcemap=external --external @huggingface/transformers --external ellmers-core --outdir ./dist/hf-transformers ./src/hf-transformers/browser.ts",
+    "build-hf-transformers-server": "bun build --target=node --sourcemap=external --external @huggingface/transformers --external ellmers-core --outdir ./dist/hf-transformers ./src/hf-transformers/server.ts",
+    "build-hf-transformers-worker": "bun build --target=bun --sourcemap=external --external @huggingface/transformers --external ellmers-core --outdir ./dist/hf-transformers ./src/hf-transformers/worker.ts",
+    "build-tf-mediapipe": "bun run build-tf-mediapipe-browser && bun run build-tf-mediapipe-server && bun run build-tf-mediapipe-worker",
+    "build-tf-mediapipe-browser": "bun build --target=browser --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/browser.ts",
+    "build-tf-mediapipe-server": "bun build --target=node --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/server.ts",
+    "build-tf-mediapipe-worker": "bun build --target=bun --sourcemap=external --external @mediapipe/tasks-text --external ellmers-core --outdir ./dist/tf-mediapipe ./src/tf-mediapipe/worker.ts",
+    "build-types": "tsc",
+    "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "test": "bun test"
+  },
+  "exports": {
+    "./hf-transformers/browser": {
+      "import": "./dist/hf-transformers/browser.js",
+      "types": "./dist/hf-transformers/browser.d.ts"
+    },
+    "./hf-transformers/server": {
+      "import": "./dist/hf-transformers/server.js",
+      "types": "./dist/hf-transformers/server.d.ts"
+    },
+    "./hf-transformers/worker": {
+      "import": "./dist/hf-transformers/worker.js",
+      "types": "./dist/hf-transformers/worker.d.ts"
+    },
+    "./tf-mediapipe/browser": {
+      "import": "./dist/tf-mediapipe/browser.js",
+      "types": "./dist/tf-mediapipe/browser.d.ts"
+    },
+    "./tf-mediapipe/server": {
+      "import": "./dist/tf-mediapipe/server.js",
+      "types": "./dist/tf-mediapipe/server.d.ts"
+    },
+    "./tf-mediapipe/worker": {
+      "import": "./dist/tf-mediapipe/worker.js",
+      "types": "./dist/tf-mediapipe/worker.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "ellmers-core": "workspace:packages/core",
+    "ellmers-task-llm": "workspace:packages/task-llm"
+  }
+}

packages/provider-tf-mediapipe/src/server.ts → packages/provider/src/hf-transformers/server.ts

@@ -5,4 +5,4 @@
 //    *   Licensed under the Apache License, Version 2.0 (the "License");           *
 //    *******************************************************************************
 
-export * from ".";
+export * from "./browser";