Skip to content

Commit

Permalink
fix(Truncate): ensure truncation is applied to all datasets in a cont…
Browse files Browse the repository at this point in the history
…ainer
  • Loading branch information
jrs65 committed Jan 11, 2024
1 parent 6030011 commit e7c7b22
Showing 1 changed file with 6 additions and 5 deletions.
11 changes: 6 additions & 5 deletions draco/core/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -757,13 +757,12 @@ def _get_params(self, container, dset):
"""
# Check if dataset should get truncated at all
if (self.dataset is None) or (dset not in self.dataset):
if dset not in container._dataset_spec or not container._dataset_spec[
dset
].get("truncate", False):
cdspec = container._class_dataset_spec()
if dset not in cdspec or not cdspec[dset].get("truncate", False):
self.log.debug(f"Not truncating dataset '{dset}' in {container}.")
return None
# Use the dataset spec if nothing specified in config
given_params = container._dataset_spec[dset].get("truncate", False)
given_params = cdspec[dset].get("truncate", False)
else:
given_params = self.dataset[dset]

Expand Down Expand Up @@ -809,14 +808,16 @@ def process(self, data):
if self.ensure_chunked:
data._ensure_chunked()

for dset in data._dataset_spec:
for dset in data.dataset_spec:
# get truncation parameters from config or container defaults
specs = self._get_params(type(data), dset)

if (specs is None) or (dset not in data):
# Don't truncate this dataset
continue

self.log.debug(f"Truncating {dset}")

old_shape = data[dset][:].shape
# np.ndarray.reshape must be used with ndarrays
# MPIArrays use MPIArray.reshape()
Expand Down

0 comments on commit e7c7b22

Please sign in to comment.