Skip to content

Commit

Permalink
feat(containers): increase default chunk size for a few containers
Browse files Browse the repository at this point in the history
  • Loading branch information
@ljgray
ljgray committed Nov 13, 2024
1 parent 0139ce9 commit eb114b0
Showing 1 changed file with 28 additions and 17 deletions.
45 changes: 28 additions & 17 deletions draco/core/containers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1134,7 +1134,7 @@ class SiderealStream(
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
"truncate": {
"weight_dataset": "vis_weight",
},
Expand All @@ -1147,7 +1147,7 @@ class SiderealStream(
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
"truncate": True,
},
"input_flags": {
Expand All @@ -1171,7 +1171,7 @@ class SiderealStream(
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (3, 64, 128, 128),
"chunks": (3, 32, 512, 2048),
"truncate": True,
},
"nsample": {
Expand All @@ -1182,7 +1182,7 @@ class SiderealStream(
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
},
"effective_ra": {
"axes": ["freq", "stack", "ra"],
Expand All @@ -1192,7 +1192,7 @@ class SiderealStream(
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
"truncate": True,
},
}
Expand Down Expand Up @@ -1239,7 +1239,7 @@ class SiderealDirtyStream(FreqContainer, VisContainer, SiderealContainer):
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
"truncate": True,
},
"mask": {
Expand All @@ -1255,6 +1255,9 @@ class SiderealDirtyStream(FreqContainer, VisContainer, SiderealContainer):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (32, 1, 2048),
"truncate": False,
},
"modes": {
Expand All @@ -1263,6 +1266,8 @@ class SiderealDirtyStream(FreqContainer, VisContainer, SiderealContainer):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"truncate": False,
},
}
Expand Down Expand Up @@ -1316,7 +1321,7 @@ class SiderealBandCovariance(FreqContainer, VisContainer, SiderealContainer):
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 1, 128),
"chunks": (32, 512, 1, 2048),
"truncate": True,
},
}
Expand Down Expand Up @@ -1564,7 +1569,7 @@ class TimeStream(FreqContainer, VisContainer, TODContainer):
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
"truncate": {
"weight_dataset": "vis_weight",
},
Expand All @@ -1577,7 +1582,7 @@ class TimeStream(FreqContainer, VisContainer, TODContainer):
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
"truncate": True,
},
"input_flags": {
Expand Down Expand Up @@ -1621,7 +1626,7 @@ class FactorizedTimeStream(TimeStream):
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (64, 128, 128),
"chunks": (32, 512, 2048),
"truncate": True,
},
"vis_weight": {
Expand All @@ -1630,6 +1635,9 @@ class FactorizedTimeStream(TimeStream):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (512, 2048),
"truncate": False,
},
"modes": {
Expand All @@ -1638,6 +1646,9 @@ class FactorizedTimeStream(TimeStream):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"chunks": (512, 512),
"truncate": False,
},
}
Expand Down Expand Up @@ -2046,7 +2057,7 @@ class VisGridStream(FreqContainer, SiderealContainer, VisBase):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"chunks": (1, 64, 1, 64, 128),
"chunks": (1, 32, 1, 256, 2048),
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"truncate": {
Expand All @@ -2059,7 +2070,7 @@ class VisGridStream(FreqContainer, SiderealContainer, VisBase):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"chunks": (1, 64, 1, 64, 128),
"chunks": (1, 32, 1, 256, 2048),
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"truncate": True,
Expand All @@ -2069,7 +2080,7 @@ class VisGridStream(FreqContainer, SiderealContainer, VisBase):
"dtype": np.int32,
"initialise": True,
"distributed": False,
"chunks": (1, 64, 1, 64, 128),
"chunks": (1, 1, 256, 2048),
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
},
Expand Down Expand Up @@ -2284,7 +2295,7 @@ class RingMap(FreqContainer, SiderealContainer, DataWeightContainer):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"chunks": (1, 1, 64, 128, 128),
"chunks": (1, 1, 32, 2048, 512),
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"truncate": {
Expand All @@ -2297,7 +2308,7 @@ class RingMap(FreqContainer, SiderealContainer, DataWeightContainer):
"initialise": True,
"distributed": True,
"distributed_axis": "freq",
"chunks": (1, 64, 128, 128),
"chunks": (1, 32, 2048, 512),
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"truncate": True,
Expand All @@ -2308,7 +2319,7 @@ class RingMap(FreqContainer, SiderealContainer, DataWeightContainer):
"initialise": False,
"distributed": True,
"distributed_axis": "freq",
"chunks": (1, 1, 64, 128, 128),
"chunks": (1, 1, 32, 2048, 512),
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"truncate": True,
Expand All @@ -2319,7 +2330,7 @@ class RingMap(FreqContainer, SiderealContainer, DataWeightContainer):
"initialise": False,
"distributed": True,
"distributed_axis": "freq",
"chunks": (4, 512, 512),
"chunks": (4, 512, 2048),
"compression": COMPRESSION,
"compression_opts": COMPRESSION_OPTS,
"truncate": True,
Expand Down

0 comments on commit eb114b0

Please sign in to comment.