diff --git a/src/dflow/python/opio.py b/src/dflow/python/opio.py index 2ee946c3..e75991ef 100644 --- a/src/dflow/python/opio.py +++ b/src/dflow/python/opio.py @@ -33,8 +33,11 @@ def dataset(self): def __deepcopy__(self, memo=None): return self + def is_none(self): + return self.dataset.attrs.get("type") == "null" + def get_data(self): - if self.dataset.attrs.get("type") == "null": + if self.is_none(): return None data = self.dataset[()] if self.dataset.attrs.get("dtype") == "utf-8": diff --git a/src/dflow/python/utils.py b/src/dflow/python/utils.py index c4204752..7a055670 100644 --- a/src/dflow/python/utils.py +++ b/src/dflow/python/utils.py @@ -101,7 +101,10 @@ def handle_input_artifact(name, sign, slices=None, data_root="/tmp", res = [] for path in path_object: f = h5py.File(path, "r") - datasets = {k: HDF5Dataset(f, k) for k in f.keys()} + datasets = {} + for k in f.keys(): + d = HDF5Dataset(f, k) + datasets[k] = None if d.is_none() else d if set(datasets.keys()) == {str(i) for i in range(len(datasets))} \ and isinstance(res, list): # concat when all datasets are lists