Update datasources.py (#93)

* Update datasources.py Removing Hardcoded limit to pdf text. We kept it for a demo but now is undocumented behavior * bump version --------- Co-authored-by: Matthew Russo <[email protected]>
mitdbg · Jan 30, 2025 · cb72e4e · cb72e4e
1 parent c4183e0
commit cb72e4e
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -16,7 +16,7 @@
 project = "Palimpzest"
 copyright = "2025, MIT Data Systems Group"
 author = "MIT Data Systems Group"
-release = "0.5.1"
+release = "0.5.2"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "palimpzest"
-version = "0.5.1"  # if you update this, be sure to update package version in `docs/source/conf.py` as well
+version = "0.5.2"  # if you update this, be sure to update package version in `docs/source/conf.py` as well
 description = "Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language"
 readme = "README.md"
 requires-python = ">=3.8"

diff --git a/src/palimpzest/core/data/datasources.py b/src/palimpzest/core/data/datasources.py
@@ -287,7 +287,7 @@ def get_item(self, idx: int) -> DataRecord:
         dr = DataRecord(self.schema, source_id=filepath)
         dr.filename = pdf_filename
         dr.contents = pdf_bytes
-        dr.text_contents = text_content[:15000]  # TODO Very hacky
+        dr.text_contents = text_content
 
         return dr