diff --git a/.changeset/tender-swans-call.md b/.changeset/tender-swans-call.md new file mode 100644 index 0000000000..9e1cf79a57 --- /dev/null +++ b/.changeset/tender-swans-call.md @@ -0,0 +1,8 @@ +--- +"@llamaindex/core": patch +"llamaindex": patch +"@llamaindex/azure": patch +"@llamaindex/llamaindex-test": patch +--- + +refactor: @llamaindex/azure diff --git a/packages/core/src/vector-store/index.ts b/packages/core/src/vector-store/index.ts index 151e89fa73..2cec54f211 100644 --- a/packages/core/src/vector-store/index.ts +++ b/packages/core/src/vector-store/index.ts @@ -119,3 +119,31 @@ export abstract class BaseVectorStore { this.embedModel = params?.embeddingModel ?? Settings.embedModel; } } + +export const parsePrimitiveValue = ( + value?: MetadataFilterValue, +): string | number => { + if (typeof value !== "number" && typeof value !== "string") { + throw new Error("Value must be a string or number"); + } + return value; +}; + +export const parseArrayValue = ( + value?: MetadataFilterValue, +): string[] | number[] => { + const isPrimitiveArray = + Array.isArray(value) && + value.every((v) => typeof v === "string" || typeof v === "number"); + if (!isPrimitiveArray) { + throw new Error("Value must be an array of strings or numbers"); + } + return value; +}; + +export const parseNumberValue = (value?: MetadataFilterValue): number => { + if (typeof value !== "number") throw new Error("Value must be a number"); + return value; +}; + +export * from "./utils.js"; diff --git a/packages/llamaindex/src/vector-store/utils.ts b/packages/core/src/vector-store/utils.ts similarity index 71% rename from packages/llamaindex/src/vector-store/utils.ts rename to packages/core/src/vector-store/utils.ts index ac50a43e0d..02c16afbed 100644 --- a/packages/llamaindex/src/vector-store/utils.ts +++ b/packages/core/src/vector-store/utils.ts @@ -1,6 +1,9 @@ -import type { BaseNode, Metadata } from "@llamaindex/core/schema"; -import { ObjectType, jsonToNode } from "@llamaindex/core/schema"; -import type { MetadataFilterValue } from "@llamaindex/core/vector-store"; +import { + ObjectType, + jsonToNode, + type BaseNode, + type Metadata, +} from "../schema"; const DEFAULT_TEXT_KEY = "text"; @@ -91,32 +94,6 @@ export function metadataDictToNode( } } -export const parsePrimitiveValue = ( - value?: MetadataFilterValue, -): string | number => { - if (typeof value !== "number" && typeof value !== "string") { - throw new Error("Value must be a string or number"); - } - return value; -}; - -export const parseArrayValue = ( - value?: MetadataFilterValue, -): string[] | number[] => { - const isPrimitiveArray = - Array.isArray(value) && - value.every((v) => typeof v === "string" || typeof v === "number"); - if (!isPrimitiveArray) { - throw new Error("Value must be an array of strings or numbers"); - } - return value; -}; - -export const parseNumberValue = (value?: MetadataFilterValue): number => { - if (typeof value !== "number") throw new Error("Value must be a number"); - return value; -}; - export const escapeLikeString = (value: string) => { return value.replace(/[%_\\]/g, "\\$&"); }; diff --git a/packages/llamaindex/package.json b/packages/llamaindex/package.json index 01d250cac0..fb3d949802 100644 --- a/packages/llamaindex/package.json +++ b/packages/llamaindex/package.json @@ -23,9 +23,6 @@ "@anthropic-ai/sdk": "0.32.1", "@aws-crypto/sha256-js": "^5.2.0", "@aws-sdk/client-sso-oidc": "^3.693.0", - "@azure/cosmos": "^4.1.1", - "@azure/identity": "^4.4.1", - "@azure/search-documents": "^12.1.0", "@datastax/astra-db-ts": "^1.4.1", "@discoveryjs/json-ext": "^0.6.1", "@google-cloud/vertexai": "1.9.0", @@ -47,6 +44,7 @@ "@llamaindex/replicate": "workspace:*", "@llamaindex/vllm": "workspace:*", "@llamaindex/postgres": "workspace:*", + "@llamaindex/azure": "workspace:*", "@mistralai/mistralai": "^1.3.4", "@mixedbread-ai/sdk": "^2.2.11", "@pinecone-database/pinecone": "^4.0.0", diff --git a/packages/llamaindex/src/index.ts b/packages/llamaindex/src/index.ts index 25231f3761..e2a71652c6 100644 --- a/packages/llamaindex/src/index.ts +++ b/packages/llamaindex/src/index.ts @@ -11,8 +11,8 @@ export { type VertexGeminiSessionOptions } from "./llm/gemini/types.js"; export { GeminiVertexSession } from "./llm/gemini/vertex.js"; // Expose AzureDynamicSessionTool for node.js runtime only +export { AzureDynamicSessionTool } from "@llamaindex/azure"; export { JinaAIEmbedding } from "./embeddings/JinaAIEmbedding.js"; -export { AzureDynamicSessionTool } from "./tools/AzureDynamicSessionTool.node.js"; // Don't export vector store modules for non-node.js runtime on top level, // as we cannot guarantee that they will work in other environments diff --git a/packages/llamaindex/src/storage/index.ts b/packages/llamaindex/src/storage/index.ts index 5ed611e6e1..4249618074 100644 --- a/packages/llamaindex/src/storage/index.ts +++ b/packages/llamaindex/src/storage/index.ts @@ -1,3 +1,4 @@ +export * from "@llamaindex/azure/storage"; export * from "@llamaindex/core/storage/chat-store"; export * from "@llamaindex/core/storage/doc-store"; export * from "@llamaindex/core/storage/index-store"; @@ -7,14 +8,7 @@ export { PostgresIndexStore, PostgresKVStore, } from "@llamaindex/postgres"; -export * from "./chatStore/AzureCosmosMongovCoreChatStore.js"; -export * from "./chatStore/AzureCosmosNoSqlChatStore.js"; -export * from "./docStore/AzureCosmosMongovCoreDocumentStore.js"; -export * from "./docStore/AzureCosmosNoSqlDocumentStore.js"; export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js"; export * from "./FileSystem.js"; -export * from "./indexStore/AzureCosmosMongovCoreIndexStore.js"; -export * from "./indexStore/AzureCosmosNoSqlIndexStore.js"; -export * from "./kvStore/AzureCosmosMongovCoreKVStore.js"; -export * from "./kvStore/AzureCosmosNoSqlKVStore.js"; + export * from "./StorageContext.js"; diff --git a/packages/llamaindex/src/tools/ToolsFactory.ts b/packages/llamaindex/src/tools/ToolsFactory.ts index 8c57380523..31ca1b7f44 100644 --- a/packages/llamaindex/src/tools/ToolsFactory.ts +++ b/packages/llamaindex/src/tools/ToolsFactory.ts @@ -1,7 +1,7 @@ import { AzureDynamicSessionTool, type AzureDynamicSessionToolParams, -} from "./AzureDynamicSessionTool.node.js"; +} from "@llamaindex/azure"; import { WikipediaTool, type WikipediaToolParams } from "./WikipediaTool.js"; // eslint-disable-next-line @typescript-eslint/no-namespace diff --git a/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts b/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts index 3f366305f3..5193aeba6b 100644 --- a/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts +++ b/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts @@ -12,6 +12,9 @@ import { BaseVectorStore, FilterCondition, FilterOperator, + metadataDictToNode, + nodeToMetadata, + parseArrayValue, type MetadataFilter, type MetadataFilters, type VectorStoreBaseParams, @@ -19,11 +22,6 @@ import { type VectorStoreQueryResult, } from "@llamaindex/core/vector-store"; import { getEnv } from "@llamaindex/env"; -import { - metadataDictToNode, - nodeToMetadata, - parseArrayValue, -} from "./utils.js"; export class AstraDBVectorStore extends BaseVectorStore { storesText: boolean = true; diff --git a/packages/llamaindex/src/vector-store/ChromaVectorStore.ts b/packages/llamaindex/src/vector-store/ChromaVectorStore.ts index e8b9426f11..bdda573a59 100644 --- a/packages/llamaindex/src/vector-store/ChromaVectorStore.ts +++ b/packages/llamaindex/src/vector-store/ChromaVectorStore.ts @@ -4,6 +4,8 @@ import { BaseVectorStore, FilterCondition, FilterOperator, + metadataDictToNode, + nodeToMetadata, VectorStoreQueryMode, type MetadataFilters, type VectorStoreBaseParams, @@ -19,7 +21,6 @@ import { type Where, type WhereDocument, } from "chromadb"; -import { metadataDictToNode, nodeToMetadata } from "./utils.js"; type ChromaDeleteOptions = { where?: Where; diff --git a/packages/llamaindex/src/vector-store/MilvusVectorStore.ts b/packages/llamaindex/src/vector-store/MilvusVectorStore.ts index 97b7029f21..5fcd56ba05 100644 --- a/packages/llamaindex/src/vector-store/MilvusVectorStore.ts +++ b/packages/llamaindex/src/vector-store/MilvusVectorStore.ts @@ -2,6 +2,10 @@ import type { ChannelOptions } from "@grpc/grpc-js"; import { BaseNode, MetadataMode, type Metadata } from "@llamaindex/core/schema"; import { BaseVectorStore, + metadataDictToNode, + nodeToMetadata, + parseArrayValue, + parsePrimitiveValue, type MetadataFilters, type VectorStoreBaseParams, type VectorStoreQuery, @@ -16,12 +20,6 @@ import { type RowData, type SearchSimpleReq, } from "@zilliz/milvus2-sdk-node"; -import { - metadataDictToNode, - nodeToMetadata, - parseArrayValue, - parsePrimitiveValue, -} from "./utils.js"; function parseScalarFilters(scalarFilters: MetadataFilters): string { const condition = scalarFilters.condition ?? "and"; diff --git a/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts b/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts index 3aaef9dda8..4fcd1d9fe8 100644 --- a/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts +++ b/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts @@ -4,6 +4,8 @@ import { MetadataMode } from "@llamaindex/core/schema"; import { BaseVectorStore, FilterCondition, + metadataDictToNode, + nodeToMetadata, type FilterOperator, type MetadataFilter, type MetadataFilters, @@ -14,7 +16,6 @@ import { import { getEnv } from "@llamaindex/env"; import type { BulkWriteOptions, Collection } from "mongodb"; import { MongoClient } from "mongodb"; -import { metadataDictToNode, nodeToMetadata } from "./utils.js"; // define your Atlas Search index. See detail https://www.mongodb.com/docs/atlas/atlas-search/field-types/knn-vector/ const DEFAULT_EMBEDDING_DEFINITION = { diff --git a/packages/llamaindex/src/vector-store/PineconeVectorStore.ts b/packages/llamaindex/src/vector-store/PineconeVectorStore.ts index e5cce0bff4..83e335296f 100644 --- a/packages/llamaindex/src/vector-store/PineconeVectorStore.ts +++ b/packages/llamaindex/src/vector-store/PineconeVectorStore.ts @@ -10,6 +10,10 @@ import { } from "@llamaindex/core/vector-store"; import type { BaseNode, Metadata } from "@llamaindex/core/schema"; +import { + metadataDictToNode, + nodeToMetadata, +} from "@llamaindex/core/vector-store"; import { getEnv } from "@llamaindex/env"; import type { FetchResponse, @@ -19,7 +23,6 @@ import type { ScoredPineconeRecord, } from "@pinecone-database/pinecone"; import { type Pinecone } from "@pinecone-database/pinecone"; -import { metadataDictToNode, nodeToMetadata } from "./utils.js"; type PineconeParams = { indexName?: string; diff --git a/packages/llamaindex/src/vector-store/QdrantVectorStore.ts b/packages/llamaindex/src/vector-store/QdrantVectorStore.ts index 7a7c083f5a..93e86ba2d7 100644 --- a/packages/llamaindex/src/vector-store/QdrantVectorStore.ts +++ b/packages/llamaindex/src/vector-store/QdrantVectorStore.ts @@ -9,9 +9,12 @@ import { type VectorStoreQueryResult, } from "@llamaindex/core/vector-store"; +import { + metadataDictToNode, + nodeToMetadata, +} from "@llamaindex/core/vector-store"; import type { QdrantClientParams, Schemas } from "@qdrant/js-client-rest"; import { QdrantClient } from "@qdrant/js-client-rest"; -import { metadataDictToNode, nodeToMetadata } from "./utils.js"; type QdrantFilter = Schemas["Filter"]; type QdrantMustConditions = QdrantFilter["must"]; diff --git a/packages/llamaindex/src/vector-store/SimpleVectorStore.ts b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts index a0ab5b3f53..b68a6ac7d3 100644 --- a/packages/llamaindex/src/vector-store/SimpleVectorStore.ts +++ b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts @@ -8,6 +8,9 @@ import type { BaseNode } from "@llamaindex/core/schema"; import { BaseVectorStore, FilterOperator, + nodeToMetadata, + parseArrayValue, + parsePrimitiveValue, VectorStoreQueryMode, type MetadataFilter, type MetadataFilters, @@ -17,11 +20,6 @@ import { } from "@llamaindex/core/vector-store"; import { fs, path } from "@llamaindex/env"; import { exists } from "../storage/FileSystem.js"; -import { - nodeToMetadata, - parseArrayValue, - parsePrimitiveValue, -} from "./utils.js"; const LEARNER_MODES = new Set([ VectorStoreQueryMode.SVM, diff --git a/packages/llamaindex/src/vector-store/UpstashVectorStore.ts b/packages/llamaindex/src/vector-store/UpstashVectorStore.ts index 5101617beb..d41edbcb32 100644 --- a/packages/llamaindex/src/vector-store/UpstashVectorStore.ts +++ b/packages/llamaindex/src/vector-store/UpstashVectorStore.ts @@ -8,9 +8,12 @@ import { } from "@llamaindex/core/vector-store"; import type { BaseNode, Metadata, TextNode } from "@llamaindex/core/schema"; +import { + metadataDictToNode, + nodeToMetadata, +} from "@llamaindex/core/vector-store"; import { getEnv } from "@llamaindex/env"; import { Index } from "@upstash/vector"; -import { metadataDictToNode, nodeToMetadata } from "./utils.js"; type UpstashParams = { namespace?: string; diff --git a/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts b/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts index c85dccaf07..42ba823de4 100644 --- a/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts +++ b/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts @@ -10,6 +10,10 @@ import weaviate, { import { BaseVectorStore, + metadataDictToNode, + nodeToMetadata, + parseArrayValue, + parseNumberValue, VectorStoreQueryMode, type MetadataFilter, type MetadataFilters, @@ -19,12 +23,6 @@ import { } from "@llamaindex/core/vector-store"; import { getEnv } from "@llamaindex/env"; import type { BaseHybridOptions } from "weaviate-client"; -import { - metadataDictToNode, - nodeToMetadata, - parseArrayValue, - parseNumberValue, -} from "./utils.js"; const NODE_SCHEMA = [ { diff --git a/packages/llamaindex/src/vector-store/index.ts b/packages/llamaindex/src/vector-store/index.ts index 030319e3e1..fd2bdf4320 100644 --- a/packages/llamaindex/src/vector-store/index.ts +++ b/packages/llamaindex/src/vector-store/index.ts @@ -1,3 +1,4 @@ +export * from "@llamaindex/azure"; export * from "@llamaindex/core/vector-store"; export { DEFAULT_DIMENSIONS, @@ -7,9 +8,6 @@ export { type PGVectorStoreConfig, } from "@llamaindex/postgres"; export * from "./AstraDBVectorStore.js"; -export * from "./azure/AzureAISearchVectorStore.js"; -export * from "./AzureCosmosDBMongoVectorStore.js"; -export * from "./AzureCosmosDBNoSqlVectorStore.js"; export * from "./ChromaVectorStore.js"; export * from "./MilvusVectorStore.js"; export * from "./MongoDBAtlasVectorStore.js"; diff --git a/packages/llamaindex/tests/VectorStore.test.ts b/packages/llamaindex/tests/VectorStore.test.ts index 443da2de15..0642288d95 100644 --- a/packages/llamaindex/tests/VectorStore.test.ts +++ b/packages/llamaindex/tests/VectorStore.test.ts @@ -2,7 +2,7 @@ import { Document, MetadataMode } from "@llamaindex/core/schema"; import { metadataDictToNode, nodeToMetadata, -} from "llamaindex/vector-store/utils"; +} from "@llamaindex/core/vector-store"; import { beforeEach, describe, expect, test } from "vitest"; describe("Testing VectorStore utils", () => { diff --git a/packages/llamaindex/tests/package.json b/packages/llamaindex/tests/package.json index 29598ef758..f5ce66a6c9 100644 --- a/packages/llamaindex/tests/package.json +++ b/packages/llamaindex/tests/package.json @@ -7,6 +7,9 @@ "test": "vitest run" }, "devDependencies": { + "@azure/cosmos": "^4.1.1", + "@azure/identity": "^4.4.1", + "@azure/search-documents": "^12.1.0", "@faker-js/faker": "^9.2.0", "dotenv": "^16.4.5", "llamaindex": "workspace:*", diff --git a/packages/providers/storage/azure/package.json b/packages/providers/storage/azure/package.json new file mode 100644 index 0000000000..a79d5b7aab --- /dev/null +++ b/packages/providers/storage/azure/package.json @@ -0,0 +1,70 @@ +{ + "name": "@llamaindex/azure", + "description": "Azure Storage for LlamaIndex", + "version": "0.0.1", + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "exports": { + ".": { + "edge-light": { + "types": "./dist/index.edge-light.d.ts", + "default": "./dist/index.edge-light.js" + }, + "workerd": { + "types": "./dist/index.edge-light.d.ts", + "default": "./dist/index.edge-light.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + }, + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "./storage": { + "edge-light": { + "types": "./dist/storage.edge-light.d.ts", + "default": "./dist/storage.edge-light.js" + }, + "workerd": { + "types": "./dist/storage.edge-light.d.ts", + "default": "./dist/storage.edge-light.js" + }, + "require": { + "types": "./dist/storage.d.cts", + "default": "./dist/storage.cjs" + }, + "import": { + "types": "./dist/storage.d.ts", + "default": "./dist/storage.js" + } + } + }, + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "https://github.com/run-llama/LlamaIndexTS.git", + "directory": "packages/providers/storage/azure" + }, + "scripts": { + "build": "bunchee", + "dev": "bunchee --watch" + }, + "devDependencies": { + "bunchee": "6.2.0", + "@types/node": "^22.9.0" + }, + "dependencies": { + "@llamaindex/core": "workspace:*", + "@llamaindex/env": "workspace:*", + "@azure/cosmos": "^4.1.1", + "@azure/identity": "^4.4.1", + "@azure/search-documents": "^12.1.0", + "mongodb": "^6.7.0" + } +} diff --git a/packages/llamaindex/src/storage/chatStore/AzureCosmosMongovCoreChatStore.ts b/packages/providers/storage/azure/src/chatStore/AzureCosmosMongovCoreChatStore.ts similarity index 100% rename from packages/llamaindex/src/storage/chatStore/AzureCosmosMongovCoreChatStore.ts rename to packages/providers/storage/azure/src/chatStore/AzureCosmosMongovCoreChatStore.ts diff --git a/packages/llamaindex/src/storage/chatStore/AzureCosmosNoSqlChatStore.ts b/packages/providers/storage/azure/src/chatStore/AzureCosmosNoSqlChatStore.ts similarity index 100% rename from packages/llamaindex/src/storage/chatStore/AzureCosmosNoSqlChatStore.ts rename to packages/providers/storage/azure/src/chatStore/AzureCosmosNoSqlChatStore.ts diff --git a/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts b/packages/providers/storage/azure/src/docStore/AzureCosmosMongovCoreDocumentStore.ts similarity index 100% rename from packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts rename to packages/providers/storage/azure/src/docStore/AzureCosmosMongovCoreDocumentStore.ts diff --git a/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts b/packages/providers/storage/azure/src/docStore/AzureCosmosNoSqlDocumentStore.ts similarity index 100% rename from packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts rename to packages/providers/storage/azure/src/docStore/AzureCosmosNoSqlDocumentStore.ts diff --git a/packages/providers/storage/azure/src/index.ts b/packages/providers/storage/azure/src/index.ts new file mode 100644 index 0000000000..d82081dd84 --- /dev/null +++ b/packages/providers/storage/azure/src/index.ts @@ -0,0 +1,9 @@ +export * from "./storage.js"; + +export * from "./vectorStore/AzureAISearchVectorStore.js"; +export * from "./vectorStore/AzureAISearchVectorStoreConfig.js"; +export * from "./vectorStore/AzureCosmosDBMongoVectorStore.js"; +export * from "./vectorStore/AzureCosmosDBNoSqlVectorStore.js"; +export * from "./vectorStore/AzureQueryResultSearch.js"; + +export * from "./tools/AzureDynamicSessionTool.node.js"; diff --git a/packages/llamaindex/src/storage/indexStore/AzureCosmosMongovCoreIndexStore.ts b/packages/providers/storage/azure/src/indexStore/AzureCosmosMongovCoreIndexStore.ts similarity index 95% rename from packages/llamaindex/src/storage/indexStore/AzureCosmosMongovCoreIndexStore.ts rename to packages/providers/storage/azure/src/indexStore/AzureCosmosMongovCoreIndexStore.ts index 2d7118d054..29d7456281 100644 --- a/packages/llamaindex/src/storage/indexStore/AzureCosmosMongovCoreIndexStore.ts +++ b/packages/providers/storage/azure/src/indexStore/AzureCosmosMongovCoreIndexStore.ts @@ -1,6 +1,6 @@ +import { KVIndexStore } from "@llamaindex/core/storage/index-store"; import { MongoClient } from "mongodb"; import { AzureCosmosVCoreKVStore } from "../kvStore/AzureCosmosMongovCoreKVStore.js"; -import { KVIndexStore } from "./KVIndexStore.js"; const DEFAULT_DATABASE = "IndexStoreDB"; const DEFAULT_COLLECTION = "IndexStoreCollection"; diff --git a/packages/llamaindex/src/storage/indexStore/AzureCosmosNoSqlIndexStore.ts b/packages/providers/storage/azure/src/indexStore/AzureCosmosNoSqlIndexStore.ts similarity index 97% rename from packages/llamaindex/src/storage/indexStore/AzureCosmosNoSqlIndexStore.ts rename to packages/providers/storage/azure/src/indexStore/AzureCosmosNoSqlIndexStore.ts index bed6419c74..26a3edf0bf 100644 --- a/packages/llamaindex/src/storage/indexStore/AzureCosmosNoSqlIndexStore.ts +++ b/packages/providers/storage/azure/src/indexStore/AzureCosmosNoSqlIndexStore.ts @@ -1,10 +1,10 @@ +import { KVIndexStore } from "@llamaindex/core/storage/index-store"; import { AzureCosmosNoSqlKVStore, type AadTokenOptions, type AccountAndKeyOptions, type ConnectionStringOptions, } from "../kvStore/AzureCosmosNoSqlKVStore.js"; -import { KVIndexStore } from "./KVIndexStore.js"; const DEFAULT_DATABASE = "IndexStoreDB"; const DEFAULT_CONTAINER = "IndexStoreContainer"; diff --git a/packages/llamaindex/src/storage/kvStore/AzureCosmosMongovCoreKVStore.ts b/packages/providers/storage/azure/src/kvStore/AzureCosmosMongovCoreKVStore.ts similarity index 100% rename from packages/llamaindex/src/storage/kvStore/AzureCosmosMongovCoreKVStore.ts rename to packages/providers/storage/azure/src/kvStore/AzureCosmosMongovCoreKVStore.ts diff --git a/packages/llamaindex/src/storage/kvStore/AzureCosmosNoSqlKVStore.ts b/packages/providers/storage/azure/src/kvStore/AzureCosmosNoSqlKVStore.ts similarity index 100% rename from packages/llamaindex/src/storage/kvStore/AzureCosmosNoSqlKVStore.ts rename to packages/providers/storage/azure/src/kvStore/AzureCosmosNoSqlKVStore.ts diff --git a/packages/providers/storage/azure/src/storage.ts b/packages/providers/storage/azure/src/storage.ts new file mode 100644 index 0000000000..78d49f68fd --- /dev/null +++ b/packages/providers/storage/azure/src/storage.ts @@ -0,0 +1,8 @@ +export * from "./chatStore/AzureCosmosMongovCoreChatStore.js"; +export * from "./chatStore/AzureCosmosNoSqlChatStore.js"; +export * from "./docStore/AzureCosmosMongovCoreDocumentStore.js"; +export * from "./docStore/AzureCosmosNoSqlDocumentStore.js"; +export * from "./indexStore/AzureCosmosMongovCoreIndexStore.js"; +export * from "./indexStore/AzureCosmosNoSqlIndexStore.js"; +export * from "./kvStore/AzureCosmosMongovCoreKVStore.js"; +export * from "./kvStore/AzureCosmosNoSqlKVStore.js"; diff --git a/packages/llamaindex/src/tools/AzureDynamicSessionTool.node.ts b/packages/providers/storage/azure/src/tools/AzureDynamicSessionTool.node.ts similarity index 100% rename from packages/llamaindex/src/tools/AzureDynamicSessionTool.node.ts rename to packages/providers/storage/azure/src/tools/AzureDynamicSessionTool.node.ts diff --git a/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts b/packages/providers/storage/azure/src/vectorStore/AzureAISearchVectorStore.ts similarity index 99% rename from packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts rename to packages/providers/storage/azure/src/vectorStore/AzureAISearchVectorStore.ts index 5dfb226861..8b7fb37327 100644 --- a/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts +++ b/packages/providers/storage/azure/src/vectorStore/AzureAISearchVectorStore.ts @@ -31,14 +31,15 @@ import { BaseVectorStore, FilterCondition, FilterOperator, + metadataDictToNode, type MetadataFilters, + nodeToMetadata, type VectorStoreBaseParams, type VectorStoreQuery, VectorStoreQueryMode, type VectorStoreQueryResult, } from "@llamaindex/core/vector-store"; import { consoleLogger, getEnv } from "@llamaindex/env"; -import { metadataDictToNode, nodeToMetadata } from "../utils.js"; import { AzureAISearchVectorStoreConfig, type R, diff --git a/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStoreConfig.ts b/packages/providers/storage/azure/src/vectorStore/AzureAISearchVectorStoreConfig.ts similarity index 100% rename from packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStoreConfig.ts rename to packages/providers/storage/azure/src/vectorStore/AzureAISearchVectorStoreConfig.ts diff --git a/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts b/packages/providers/storage/azure/src/vectorStore/AzureCosmosDBMongoVectorStore.ts similarity index 99% rename from packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts rename to packages/providers/storage/azure/src/vectorStore/AzureCosmosDBMongoVectorStore.ts index 4870273267..b0b0108745 100644 --- a/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts +++ b/packages/providers/storage/azure/src/vectorStore/AzureCosmosDBMongoVectorStore.ts @@ -2,13 +2,14 @@ import type { BaseNode } from "@llamaindex/core/schema"; import { MetadataMode } from "@llamaindex/core/schema"; import { BaseVectorStore, + metadataDictToNode, + nodeToMetadata, type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, } from "@llamaindex/core/vector-store"; import { getEnv } from "@llamaindex/env"; import { Collection, Db, MongoClient } from "mongodb"; -import { metadataDictToNode, nodeToMetadata } from "./utils.js"; /** Azure Cosmos DB for MongoDB vCore Similarity type. */ export const AzureCosmosDBMongoDBSimilarityType = { diff --git a/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts b/packages/providers/storage/azure/src/vectorStore/AzureCosmosDBNoSqlVectorStore.ts similarity index 99% rename from packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts rename to packages/providers/storage/azure/src/vectorStore/AzureCosmosDBNoSqlVectorStore.ts index 6118fef753..27ac379d38 100644 --- a/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts +++ b/packages/providers/storage/azure/src/vectorStore/AzureCosmosDBNoSqlVectorStore.ts @@ -12,8 +12,11 @@ import { } from "@azure/cosmos"; import { DefaultAzureCredential, type TokenCredential } from "@azure/identity"; import { BaseNode, MetadataMode } from "@llamaindex/core/schema"; +import { + metadataDictToNode, + nodeToMetadata, +} from "@llamaindex/core/vector-store"; import { getEnv } from "@llamaindex/env"; -import { metadataDictToNode, nodeToMetadata } from "./utils.js"; import { BaseVectorStore, diff --git a/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts b/packages/providers/storage/azure/src/vectorStore/AzureQueryResultSearch.ts similarity index 99% rename from packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts rename to packages/providers/storage/azure/src/vectorStore/AzureQueryResultSearch.ts index 721e23f6d6..fb554db9a7 100644 --- a/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts +++ b/packages/providers/storage/azure/src/vectorStore/AzureQueryResultSearch.ts @@ -9,8 +9,8 @@ import { } from "@llamaindex/core/vector-store"; import type { TextNode } from "@llamaindex/core/schema"; +import { metadataDictToNode } from "@llamaindex/core/vector-store"; import { consoleLogger } from "@llamaindex/env"; -import { metadataDictToNode } from "../utils.js"; import { AzureAISearchVectorStoreConfig, type R, diff --git a/packages/providers/storage/azure/tsconfig.json b/packages/providers/storage/azure/tsconfig.json new file mode 100644 index 0000000000..d3fa4b6719 --- /dev/null +++ b/packages/providers/storage/azure/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../../../tsconfig.json", + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "outDir": "./lib", + "tsBuildInfoFile": "./lib/.tsbuildinfo", + "types": ["node"] + }, + "include": ["./src"], + "references": [ + { + "path": "../../../core/tsconfig.json" + }, + { + "path": "../../../env/tsconfig.json" + } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 041d4c3e76..6fc94973e5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -941,15 +941,6 @@ importers: '@aws-sdk/client-sso-oidc': specifier: ^3.693.0 version: 3.693.0(@aws-sdk/client-sts@3.714.0) - '@azure/cosmos': - specifier: ^4.1.1 - version: 4.1.1 - '@azure/identity': - specifier: ^4.4.1 - version: 4.4.1 - '@azure/search-documents': - specifier: ^12.1.0 - version: 12.1.0 '@datastax/astra-db-ts': specifier: ^1.4.1 version: 1.4.1 @@ -968,6 +959,9 @@ importers: '@llamaindex/anthropic': specifier: workspace:* version: link:../providers/anthropic + '@llamaindex/azure': + specifier: workspace:* + version: link:../providers/storage/azure '@llamaindex/clip': specifier: workspace:* version: link:../providers/clip @@ -1110,6 +1104,15 @@ importers: packages/llamaindex/tests: devDependencies: + '@azure/cosmos': + specifier: ^4.1.1 + version: 4.1.1 + '@azure/identity': + specifier: ^4.4.1 + version: 4.4.1 + '@azure/search-documents': + specifier: ^12.1.0 + version: 12.1.0 '@faker-js/faker': specifier: ^9.2.0 version: 9.2.0 @@ -1316,6 +1319,34 @@ importers: specifier: 6.2.0 version: 6.2.0(typescript@5.7.2) + packages/providers/storage/azure: + dependencies: + '@azure/cosmos': + specifier: ^4.1.1 + version: 4.1.1 + '@azure/identity': + specifier: ^4.4.1 + version: 4.4.1 + '@azure/search-documents': + specifier: ^12.1.0 + version: 12.1.0 + '@llamaindex/core': + specifier: workspace:* + version: link:../../../core + '@llamaindex/env': + specifier: workspace:* + version: link:../../../env + mongodb: + specifier: ^6.7.0 + version: 6.12.0(@aws-sdk/credential-providers@3.714.0) + devDependencies: + '@types/node': + specifier: ^22.9.0 + version: 22.9.0 + bunchee: + specifier: 6.2.0 + version: 6.2.0(typescript@5.7.2) + packages/providers/storage/postgres: dependencies: '@llamaindex/core': @@ -13413,7 +13444,7 @@ snapshots: jws: 4.0.0 open: 8.4.2 stoppable: 1.1.0 - tslib: 2.6.0 + tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -13444,7 +13475,7 @@ snapshots: '@azure/core-util': 1.11.0 '@azure/logger': 1.1.4 events: 3.3.0 - tslib: 2.6.0 + tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -13469,7 +13500,7 @@ snapshots: '@babel/traverse': 7.26.4 '@babel/types': 7.26.3 convert-source-map: 2.0.0 - debug: 4.3.4 + debug: 4.4.0 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13558,7 +13589,7 @@ snapshots: '@babel/parser': 7.26.3 '@babel/template': 7.25.9 '@babel/types': 7.26.3 - debug: 4.3.4 + debug: 4.4.0 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -13601,7 +13632,7 @@ snapshots: outdent: 0.5.0 prettier: 2.8.8 resolve-from: 5.0.0 - semver: 7.5.4 + semver: 7.6.3 '@changesets/assemble-release-plan@6.0.5': dependencies: @@ -13610,7 +13641,7 @@ snapshots: '@changesets/should-skip-package': 0.1.1 '@changesets/types': 6.0.0 '@manypkg/get-packages': 1.1.3 - semver: 7.5.4 + semver: 7.6.3 '@changesets/changelog-git@0.2.0': dependencies: @@ -13671,7 +13702,7 @@ snapshots: '@changesets/types': 6.0.0 '@manypkg/get-packages': 1.1.3 picocolors: 1.1.1 - semver: 7.5.4 + semver: 7.6.3 '@changesets/get-release-plan@4.0.6': dependencies: @@ -14131,7 +14162,7 @@ snapshots: '@eslint/config-array@0.19.1': dependencies: '@eslint/object-schema': 2.1.5 - debug: 4.3.4 + debug: 4.4.0 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -14143,7 +14174,7 @@ snapshots: '@eslint/eslintrc@3.2.0': dependencies: ajv: 6.12.6 - debug: 4.3.4 + debug: 4.4.0 espree: 10.3.0 globals: 14.0.0 ignore: 5.2.4 @@ -14522,7 +14553,7 @@ snapshots: nopt: 5.0.0 npmlog: 5.0.1 rimraf: 3.0.2 - semver: 7.5.4 + semver: 7.6.3 tar: 6.2.1 transitivePeerDependencies: - encoding @@ -16401,7 +16432,7 @@ snapshots: '@typescript-eslint/scope-manager': 5.59.2 '@typescript-eslint/types': 5.59.2 '@typescript-eslint/typescript-estree': 5.59.2(typescript@5.7.2) - debug: 4.3.4 + debug: 4.4.0 eslint: 9.16.0(jiti@2.4.2) optionalDependencies: typescript: 5.7.2 @@ -16414,7 +16445,7 @@ snapshots: '@typescript-eslint/types': 8.18.0 '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.7.2) '@typescript-eslint/visitor-keys': 8.18.0 - debug: 4.3.4 + debug: 4.4.0 eslint: 9.16.0(jiti@2.4.2) typescript: 5.7.2 transitivePeerDependencies: @@ -16439,7 +16470,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.7.2) '@typescript-eslint/utils': 8.18.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2) - debug: 4.3.4 + debug: 4.4.0 eslint: 9.16.0(jiti@2.4.2) ts-api-utils: 1.4.3(typescript@5.7.2) typescript: 5.7.2 @@ -16450,7 +16481,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 8.18.1(typescript@5.7.2) '@typescript-eslint/utils': 8.18.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2) - debug: 4.3.4 + debug: 4.4.0 eslint: 9.16.0(jiti@2.4.2) ts-api-utils: 1.4.3(typescript@5.7.2) typescript: 5.7.2 @@ -16469,10 +16500,10 @@ snapshots: dependencies: '@typescript-eslint/types': 5.59.2 '@typescript-eslint/visitor-keys': 5.59.2 - debug: 4.3.4 + debug: 4.4.0 globby: 11.1.0 is-glob: 4.0.3 - semver: 7.5.4 + semver: 7.6.3 tsutils: 3.21.0(typescript@5.7.2) optionalDependencies: typescript: 5.7.2 @@ -16483,7 +16514,7 @@ snapshots: dependencies: '@typescript-eslint/types': 7.18.0 '@typescript-eslint/visitor-keys': 7.18.0 - debug: 4.3.4 + debug: 4.4.0 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.5 @@ -16498,7 +16529,7 @@ snapshots: dependencies: '@typescript-eslint/types': 8.18.0 '@typescript-eslint/visitor-keys': 8.18.0 - debug: 4.3.4 + debug: 4.4.0 fast-glob: 3.3.2 is-glob: 4.0.3 minimatch: 9.0.5 @@ -16512,7 +16543,7 @@ snapshots: dependencies: '@typescript-eslint/types': 8.18.1 '@typescript-eslint/visitor-keys': 8.18.1 - debug: 4.3.4 + debug: 4.4.0 fast-glob: 3.3.2 is-glob: 4.0.3 minimatch: 9.0.5 @@ -16566,7 +16597,7 @@ snapshots: '@typescript/vfs@1.6.0(typescript@5.7.2)': dependencies: - debug: 4.3.4 + debug: 4.4.0 typescript: 5.7.2 transitivePeerDependencies: - supports-color @@ -16834,7 +16865,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.4 + debug: 4.4.0 transitivePeerDependencies: - supports-color optional: true @@ -17203,7 +17234,7 @@ snapshots: bin-version-check@5.1.0: dependencies: bin-version: 6.0.0 - semver: 7.5.4 + semver: 7.6.3 semver-truncate: 3.0.0 bin-version@6.0.0: @@ -17395,7 +17426,7 @@ snapshots: capnp-ts@0.7.0: dependencies: - debug: 4.3.4 + debug: 4.4.0 tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -18390,7 +18421,7 @@ snapshots: eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2)): dependencies: - debug: 4.3.4 + debug: 4.4.0 enhanced-resolve: 5.15.0 eslint: 9.16.0(jiti@2.4.2) eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5)(eslint@9.16.0(jiti@2.4.2)) @@ -19610,7 +19641,7 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.3 - debug: 4.3.4 + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -19649,7 +19680,7 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.4 + debug: 4.4.0 transitivePeerDependencies: - supports-color optional: true @@ -19657,7 +19688,7 @@ snapshots: https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.3 - debug: 4.3.4 + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -20120,7 +20151,7 @@ snapshots: lodash.isstring: 4.0.1 lodash.once: 4.1.1 ms: 2.1.3 - semver: 7.5.4 + semver: 7.6.3 jsx-ast-utils@3.3.3: dependencies: @@ -21197,7 +21228,7 @@ snapshots: micromark@3.2.0: dependencies: '@types/debug': 4.1.12 - debug: 4.3.4 + debug: 4.4.0 decode-named-character-reference: 1.0.2 micromark-core-commonmark: 1.1.0 micromark-factory-space: 1.1.0 @@ -21219,7 +21250,7 @@ snapshots: micromark@4.0.1: dependencies: '@types/debug': 4.1.12 - debug: 4.3.4 + debug: 4.4.0 decode-named-character-reference: 1.0.2 devlop: 1.1.0 micromark-core-commonmark: 2.0.2 @@ -21444,7 +21475,7 @@ snapshots: mquery@5.0.0: dependencies: - debug: 4.3.4 + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -21614,7 +21645,7 @@ snapshots: node-abi@3.71.0: dependencies: - semver: 7.5.4 + semver: 7.6.3 node-addon-api@6.1.0: {} @@ -23175,7 +23206,7 @@ snapshots: semver-truncate@3.0.0: dependencies: - semver: 7.5.4 + semver: 7.6.3 semver@5.7.2: {} @@ -23219,7 +23250,7 @@ snapshots: detect-libc: 2.0.3 node-addon-api: 6.1.0 prebuild-install: 7.1.2 - semver: 7.5.4 + semver: 7.6.3 simple-get: 4.0.1 tar-fs: 3.0.6 tunnel-agent: 0.6.0 diff --git a/tsconfig.json b/tsconfig.json index 5e20e187d0..7c740c1ac2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -127,6 +127,9 @@ }, { "path": "./packages/providers/storage/postgres/tsconfig.json" + }, + { + "path": "./packages/providers/storage/azure/tsconfig.json" } ] }