Skip to content

Commit

Permalink
Begin to update SQLAdapter.
Browse files Browse the repository at this point in the history
  • Loading branch information
danielballan committed Oct 4, 2024
1 parent a913d36 commit c0b88c6
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 9 deletions.
33 changes: 24 additions & 9 deletions tiled/adapters/sql.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import copy
import hashlib
import uuid
from typing import Iterator, List, Optional, Tuple, Union

import adbc_driver_postgresql.dbapi
Expand All @@ -8,7 +10,7 @@
import pyarrow.fs

from ..structures.core import Spec, StructureFamily
from ..structures.data_source import Asset
from ..structures.data_source import Asset, DataSource, Storage
from ..structures.table import TableStructure
from .array import ArrayAdapter
from .protocols import AccessPolicy
Expand Down Expand Up @@ -71,7 +73,12 @@ def metadata(self) -> JSON:
return self._metadata

@classmethod
def init_storage(cls, data_uri: str, structure: TableStructure) -> Asset:
def init_storage(
cls,
storage: Storage,
data_source: DataSource[TableStructure],
path_parts: List[str],
) -> DataSource[TableStructure]:
"""
Class to initialize the list of assets for given uri. In SQL Adapter we hve single partition.
Parameters
Expand All @@ -81,16 +88,24 @@ def init_storage(cls, data_uri: str, structure: TableStructure) -> Asset:
Returns
-------
The list of assets. In SQL Adapter we have a single asset.
A modified copy of the data source
"""
if structure.npartitions > 1:
data_source = copy.deepcopy(data_source) # Do not mutate caller input.
if data_source.structure.npartitions > 1:
raise ValueError("The SQL adapter must have only 1 partition")
return Asset(
data_uri=data_uri,
is_directory=False,
parameter="data_uris",
num=0,
default_table_name = ... # based on hash of Arrow schema
data_source.parameters.setdefault("table_name", default_table_name)
data_source.parameters["dataset_id"] = uuid.uuid4().int
data_uri = storage.sql # TODO scrub credentials
data_source.assets.append(
Asset(
data_uri=data_uri,
is_directory=False,
parameter="data_uris",
num=0,
)
)
return data_source

def structure(self) -> TableStructure:
"""
Expand Down
4 changes: 4 additions & 0 deletions tiled/catalog/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
DEFAULT_ADAPTERS_BY_MIMETYPE,
PARQUET_MIMETYPE,
SPARSE_BLOCKS_PARQUET_MIMETYPE,
TILED_SQL_TABLE_MIMETYPE,
ZARR_MIMETYPE,
)
from ..query_registration import QueryTranslationRegistry
Expand Down Expand Up @@ -115,6 +116,9 @@
APACHE_ARROW_FILE_MIME_TYPE: lambda: importlib.import_module(
"...adapters.arrow", __name__
).ArrowAdapter.init_storage,
TILED_SQL_TABLE_MIMETYPE: lambda: importlib.import_module(
"...adapters.sql", __name__
).SQLAdapter.init_storage,
}
)

Expand Down
4 changes: 4 additions & 0 deletions tiled/mimetypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
SPARSE_BLOCKS_PARQUET_MIMETYPE = "application/x-parquet;structure=sparse"
ZARR_MIMETYPE = "application/x-zarr"
AWKWARD_BUFFERS_MIMETYPE = "application/x-awkward-buffers"
TILED_SQL_TABLE_MIMETYPE = "application/x-tiled-sql-table"
DEFAULT_ADAPTERS_BY_MIMETYPE = OneShotCachedMap(
{
"image/tiff": lambda: importlib.import_module(
Expand Down Expand Up @@ -49,6 +50,9 @@
APACHE_ARROW_FILE_MIME_TYPE: lambda: importlib.import_module(
"..adapters.arrow", __name__
).ArrowAdapter,
TILED_SQL_TABLE_MIMETYPE: lambda: importlib.import_module(
"...adapters.sql", __name__
).SQLAdapter.init_storage,
}
)

Expand Down

0 comments on commit c0b88c6

Please sign in to comment.