-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Showing
72 changed files
with
119,107 additions
and
1,039,144 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| { | ||
| "actorSpecification": 1, | ||
| "name": "weaviate-integration", | ||
| "title": "Weaviate Integration", | ||
| "description": "Upload a dataset to Weaviate", | ||
| "version": "0.0", | ||
| "input": "./input_schema.json", | ||
| "dockerfile": "../../../shared/Dockerfile", | ||
| "readme": "./README.md", | ||
| "changelog":"../../../shared/CHANGELOG.md", | ||
| "storages": { | ||
| "dataset": "../../../shared/dataset_schema.json" | ||
| }, | ||
| "dockerContextDir": "../../.." | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| { | ||
| "title": "Weaviate-integration", | ||
| "type": "object", | ||
| "schemaVersion": 1, | ||
| "properties": { | ||
| "weaviateUrl": { | ||
| "title": "Weaviate URL", | ||
| "type": "string", | ||
| "description": "REST URL of the Weaviate instance to connect to", | ||
| "editor": "textfield", | ||
| "sectionCaption": "Weaviate settings" | ||
| }, | ||
| "weaviateApiKey": { | ||
| "title": "Weaviate API KEY", | ||
| "description": "Weaviate API KEY", | ||
| "type": "string", | ||
| "editor": "textfield", | ||
| "isSecret": true | ||
| }, | ||
| "weaviateCollectionName": { | ||
| "title": "Weaviate collection name", | ||
| "type": "string", | ||
| "description": "Name of the Weaviate collection where the data will be stored", | ||
| "editor": "textfield" | ||
| }, | ||
| "embeddingsProvider": { | ||
| "title": "Embeddings provider (as defined in the langchain API)", | ||
| "description": "Choose the embeddings provider to use for generating embeddings", | ||
| "type": "string", | ||
| "editor": "select", | ||
| "enum": [ | ||
| "OpenAI", | ||
| "Cohere" | ||
| ], | ||
| "default": "OpenAI", | ||
| "sectionCaption": "Embeddings settings" | ||
| }, | ||
| "embeddingsConfig": { | ||
| "title": "Configuration for embeddings provider", | ||
| "description": "Configure the parameters for the LangChain embedding class. Key points to consider:\n\n1. Typically, you only need to specify the model name. For example, for OpenAI, set the model name as {\"model\": \"text-embedding-3-small\"}.\n\n2. It's crucial to ensure that the vector size of your embeddings matches the size of embeddings in the database.\n\n3. Here are some examples of embedding models:\n - [OpenAI](https://platform.openai.com/docs/guides/embeddings): `text-embedding-3-small`, `text-embedding-3-large`, etc.\n - [Cohere](https://docs.cohere.com/docs/cohere-embed): `embed-english-v3.0`, `embed-multilingual-light-v3.0`, etc.\n\n4. For more details about other parameters, refer to the [LangChain documentation](https://python.langchain.com/v0.2/docs/integrations/text_embedding/).", | ||
| "type": "object", | ||
| "editor": "json" | ||
| }, | ||
| "embeddingsApiKey": { | ||
| "title": "Embeddings API KEY (whenever applicable, depends on provider)", | ||
| "description": "Value of the API KEY for the embeddings provider (if required).\n\n For example for OpenAI it is OPENAI_API_KEY, for Cohere it is COHERE_API_KEY)", | ||
| "type": "string", | ||
| "editor": "textfield", | ||
| "isSecret": true | ||
| }, | ||
| "datasetFields": { | ||
| "title": "Dataset fields to select from the dataset results and store in the database", | ||
| "type": "array", | ||
| "description": "This array specifies the dataset fields to be selected and stored in the vector store. Only the fields listed here will be included in the vector store.\n\nFor instance, when using the Website Content Crawler, you might choose to include fields such as `text`, `url`, and `metadata.title` in the vector store.", | ||
| "default": [ | ||
| "text" | ||
| ], | ||
| "prefill": [ | ||
| "text" | ||
| ], | ||
| "editor": "stringList", | ||
| "sectionCaption": "Dataset settings" | ||
| }, | ||
| "metadataDatasetFields": { | ||
| "title": "Dataset fields to select from the dataset and store as metadata in the database", | ||
| "type": "object", | ||
| "description": "A list of dataset fields which should be selected from the dataset and stored as metadata in the vector stores.\n\nFor example, when using the Website Content Crawler, you might want to store `url` in metadata. In this case, use `metadataDatasetFields parameter as follows {\"url\": \"url\"}`", | ||
| "editor": "json" | ||
| }, | ||
| "metadataObject": { | ||
| "title": "Custom object to be stored as metadata in the vector store database", | ||
| "type": "object", | ||
| "description": "This object allows you to store custom metadata for every item in the vector store.\n\nFor example, if you want to store the `domain` as metadata, use the `metadataObject` like this: {\"domain\": \"apify.com\"}.", | ||
| "editor": "json" | ||
| }, | ||
| "datasetId": { | ||
| "title": "Dataset ID", | ||
| "type": "string", | ||
| "description": "Dataset ID (when running standalone without integration)", | ||
| "editor": "textfield" | ||
| }, | ||
| "enableDeltaUpdates": { | ||
| "title": "Enable incremental updates for objects based on deltas", | ||
| "type": "boolean", | ||
| "description": "When set to true, this setting enables incremental updates for objects in the database by comparing the changes (deltas) between the crawled dataset items and the existing objects, uniquely identified by the `datasetKeysToItemId` field.\n\n The integration will only add new objects and update those that have changed, reducing unnecessary updates. The `datasetFields`, `metadataDatasetFields`, and `metadataObject` fields are used to determine the changes.", | ||
| "default": true, | ||
| "sectionCaption": "Delta updates settings" | ||
| }, | ||
| "deltaUpdatesPrimaryDatasetFields": { | ||
| "title": "Dataset fields to uniquely identify dataset items (only relevant when `enableDeltaUpdates` is enabled)", | ||
| "type": "array", | ||
| "items": { | ||
| "type": "string" | ||
| }, | ||
| "description": "This array contains fields that are used to uniquely identify dataset items, which helps to handle content changes across different runs.\n\nFor instance, in a web content crawling scenario, the `url` field could serve as a unique identifier for each item.", | ||
| "editor": "stringList", | ||
| "default": [ | ||
| "url" | ||
| ], | ||
| "prefill": [ | ||
| "url" | ||
| ] | ||
| }, | ||
| "expiredObjectDeletionPeriodDays": { | ||
| "title": "Delete expired objects from the database after a specified number of days (only relevant when `enableDeltaUpdates` is enabled)", | ||
| "type": "integer", | ||
| "description": "This setting allows the integration to manage the deletion of objects from the database that have not been crawled for a specified period. It is typically used in subsequent runs after the initial crawl.\n\nWhen the value is greater than 0, the integration checks if objects have been seen within the last X days (determined by the expiration period). If the objects are expired, they are deleted from the database. The specific value for `deletedExpiredObjectsDays` depends on your use case and how frequently you crawl data.\n\nFor example, if you crawl data daily, you can set `deletedExpiredObjectsDays` to 7 days. If you crawl data weekly, you can set `deletedExpiredObjectsDays` to 30 days.\n\nSetting `deletedExpiredObjectsDays` to 0 disables this feature", | ||
| "default": 30, | ||
| "minimum": 0, | ||
| "unit": "days", | ||
| "editor": "number" | ||
| }, | ||
| "performChunking": { | ||
| "title": "Enable text chunking", | ||
| "description": "When set to true, the text will be divided into smaller chunks based on the settings provided below. Proper chunking helps optimize retrieval and ensures accurate and efficient responses.", | ||
| "default": false, | ||
| "type": "boolean", | ||
| "sectionCaption": "Text chunking settings" | ||
| }, | ||
| "chunkSize": { | ||
| "title": "Maximum chunk size", | ||
| "type": "integer", | ||
| "description": "Defines the maximum number of characters in each text chunk. Choosing the right size balances between detailed context and system performance. Optimal sizes ensure high relevancy and minimal response time.", | ||
| "default": 1000, | ||
| "minimum": 1 | ||
| }, | ||
| "chunkOverlap": { | ||
| "title": "Chunk overlap", | ||
| "type": "integer", | ||
| "description": "Specifies the number of overlapping characters between consecutive text chunks. Adjusting this helps maintain context across chunks, which is crucial for accuracy in retrieval-augmented generation systems.", | ||
| "default": 0, | ||
| "minimum": 0 | ||
| } | ||
| }, | ||
| "required": [ | ||
| "weaviateUrl", | ||
| "weaviateApiKey", | ||
| "weaviateCollectionName", | ||
| "embeddingsProvider", | ||
| "embeddingsApiKey", | ||
| "datasetFields" | ||
| ] | ||
| } |
Oops, something went wrong.