diff --git a/python/setup.cfg b/python/setup.cfg index 5020d7a..a1f9616 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -38,6 +38,7 @@ dev = flake8-isort pytest-cov pytest-asyncio + h5py sphinx-rtd-theme-github-versions pydata-sphinx-theme>=0.12 sphinx-autobuild @@ -47,6 +48,8 @@ dev = tox-direct pre-commit requests + typer + pyzmq [options.entry_points] # Include a command line script @@ -75,7 +78,7 @@ extend-ignore = # Run pytest with all our checkers, and don't spam us with massive tracebacks on error addopts = --tb=native -vv --doctest-modules --doctest-glob="*.rst" - --cov=eiger_detector --cov-report term --cov-report xml:cov.xml + # --cov=eiger_detector --cov-report term --cov-report xml:cov.xml # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings filterwarnings = error # Doctest python code in docs, python code in src docstrings, test functions in tests diff --git a/python/tests/dump_zeromq_stream.py b/python/tests/dump_zeromq_stream.py new file mode 100644 index 0000000..a339c1a --- /dev/null +++ b/python/tests/dump_zeromq_stream.py @@ -0,0 +1,52 @@ +import json +from pathlib import Path +from typing import Annotated + +import typer +import zmq + +HERE = Path(__file__).parent + + +def main( + url: Annotated[ + str, typer.Argument(help="URL of ZeroMQ server") + ] = "127.0.0.1:10008", + output: Annotated[Path, typer.Option(help="Output directory")] = HERE / "input", +): + """Dump messages from a ZeroMQ PUB stream.""" + if not url.startswith("http://"): + url = f"tcp://{url}" + + context = zmq.Context() + socket = context.socket(zmq.SUB) + socket.connect(url) + socket.setsockopt_string(zmq.SUBSCRIBE, "") + + print(f"Connected to {url}") + + idx = 0 + while True: + print("Waiting for message...") + message = socket.recv() + # Check if message is json or raw bytes + try: + j = json.loads(message.decode()) + data = json.dumps(j, indent=4) + ext = ".json" + mode = "w" + except (json.decoder.JSONDecodeError, UnicodeDecodeError): + data = message + ext = "" + mode = "wb" + + with (output / f"{idx}{ext}").open(mode=mode) as f: + f.write(data) + + print(f"{idx}{ext}") + + idx += 1 + + +if __name__ == "__main__": + typer.run(main) \ No newline at end of file diff --git a/python/tests/input/00.json b/python/tests/input/00.json new file mode 100644 index 0000000..bddc496 --- /dev/null +++ b/python/tests/input/00.json @@ -0,0 +1,10 @@ +{ + "plugin": "Acquisition", + "parameter": "startacquisition", + "type": "string", + "header": { + "acqID": "test_0_0", + "rank": 0, + "totalFrames": 1 + } +} \ No newline at end of file diff --git a/python/tests/input/01 b/python/tests/input/01 new file mode 100644 index 0000000..e69de29 diff --git a/python/tests/input/02.json b/python/tests/input/02.json new file mode 100644 index 0000000..22a7df4 --- /dev/null +++ b/python/tests/input/02.json @@ -0,0 +1,10 @@ +{ + "plugin": "Acquisition", + "parameter": "createfile", + "type": "string", + "header": { + "acqID": "test_0_0", + "rank": 0, + "totalFrames": 1 + } +} \ No newline at end of file diff --git a/python/tests/input/03.json b/python/tests/input/03.json new file mode 100644 index 0000000..40f08be --- /dev/null +++ b/python/tests/input/03.json @@ -0,0 +1,4 @@ +{ + "filePath": "/dls/i03/data/2024/cm37235-3/tmp/gary/0/test_0_0_000001.h5", + "create_duration": 2071 +} \ No newline at end of file diff --git a/python/tests/input/04.json b/python/tests/input/04.json new file mode 100644 index 0000000..6713e9c --- /dev/null +++ b/python/tests/input/04.json @@ -0,0 +1,9 @@ +{ + "plugin": "eiger", + "parameter": "eiger-globalconfig", + "type": "string", + "header": { + "acqID": "test_0_0", + "series": 40181 + } +} \ No newline at end of file diff --git a/python/tests/input/05.json b/python/tests/input/05.json new file mode 100644 index 0000000..8d462ea --- /dev/null +++ b/python/tests/input/05.json @@ -0,0 +1,54 @@ +{ + "auto_summation": true, + "beam_center_x": 2047.8194396763713, + "beam_center_y": 2146.4601198541245, + "bit_depth_image": 32, + "bit_depth_readout": 16, + "chi_increment": 0.0, + "chi_start": 0.0, + "compression": "bslz4", + "count_time": 0.0999999, + "countrate_correction_applied": true, + "countrate_correction_count_cutoff": 1151122, + "data_collection_date": "2024-05-25T08:03:25.903+02:00", + "description": "Dectris EIGER2 Si 16M", + "detector_distance": 376.5651820410446, + "detector_number": "E-32-0117", + "detector_readout_time": 1e-07, + "detector_translation": [ + 0.15358645797572784, + 0.16098450898905933, + -376.5651820410446 + ], + "eiger_fw_version": "release-2022.1.2", + "element": "", + "flatfield_correction_applied": true, + "frame_count_time": 0.00344898, + "frame_period": 0.00344898, + "frame_time": 0.1, + "kappa_increment": 0.0, + "kappa_start": 0.0, + "nimages": 1, + "ntrigger": 1, + "number_of_excluded_pixels": 1253074, + "omega_increment": 0.1, + "omega_start": -90.0, + "phi_increment": 0.0, + "phi_start": 0.0, + "photon_energy": 12699.699839514962, + "pixel_mask_applied": true, + "roi_mode": "", + "sensor_material": "Si", + "sensor_thickness": 0.00045, + "software_version": "1.8.0", + "threshold_energy": 6349.849919757481, + "trigger_mode": "ints", + "two_theta_increment": 0.0, + "two_theta_start": 0.0, + "virtual_pixel_correction_applied": true, + "wavelength": 0.976276604959, + "x_pixel_size": 7.5e-05, + "x_pixels_in_detector": 4148, + "y_pixel_size": 7.5e-05, + "y_pixels_in_detector": 4362 +} \ No newline at end of file diff --git a/python/tests/input/06.json b/python/tests/input/06.json new file mode 100644 index 0000000..e314777 --- /dev/null +++ b/python/tests/input/06.json @@ -0,0 +1,13 @@ +{ + "plugin": "eiger", + "parameter": "eiger-globalflatfield", + "type": "raw", + "header": { + "acqID": "test_0_0", + "shape": [ + 4148, + 4362 + ], + "type": "float32" + } +} \ No newline at end of file diff --git a/python/tests/input/07 b/python/tests/input/07 new file mode 100644 index 0000000..67f9f3c Binary files /dev/null and b/python/tests/input/07 differ diff --git a/python/tests/input/08.json b/python/tests/input/08.json new file mode 100644 index 0000000..edb6670 --- /dev/null +++ b/python/tests/input/08.json @@ -0,0 +1,13 @@ +{ + "plugin": "eiger", + "parameter": "eiger-globalmask", + "type": "raw", + "header": { + "acqID": "test_0_0", + "shape": [ + 4148, + 4362 + ], + "type": "uint32" + } +} \ No newline at end of file diff --git a/python/tests/input/09 b/python/tests/input/09 new file mode 100644 index 0000000..14f237d Binary files /dev/null and b/python/tests/input/09 differ diff --git a/python/tests/input/10.json b/python/tests/input/10.json new file mode 100644 index 0000000..2bc33bb --- /dev/null +++ b/python/tests/input/10.json @@ -0,0 +1,13 @@ +{ + "plugin": "eiger", + "parameter": "eiger-globalcountrate", + "type": "raw", + "header": { + "acqID": "test_0_0", + "shape": [ + 2, + 1152 + ], + "type": "float32" + } +} \ No newline at end of file diff --git a/python/tests/input/11 b/python/tests/input/11 new file mode 100644 index 0000000..c0267db Binary files /dev/null and b/python/tests/input/11 differ diff --git a/python/tests/input/12.json b/python/tests/input/12.json new file mode 100644 index 0000000..b9a25b2 --- /dev/null +++ b/python/tests/input/12.json @@ -0,0 +1,21 @@ +{ + "plugin": "eiger", + "parameter": "eiger-imagedata", + "type": "string", + "header": { + "acqID": "test_0_0", + "frame": 0, + "series": 40181, + "size": 1504987, + "start_time": 3964989369387280, + "stop_time": 3964989469387180, + "real_time": 99999900, + "shape": [ + 4148, + 4362 + ], + "type": "uint32", + "encoding": "bs32-lz4<", + "hash": "3aca01d1417576a9c8c26343a93b29a1" + } +} \ No newline at end of file diff --git a/python/tests/input/13.json b/python/tests/input/13.json new file mode 100644 index 0000000..11328ce --- /dev/null +++ b/python/tests/input/13.json @@ -0,0 +1,16 @@ +{ + "acqID": "test_0_0", + "frame": 0, + "series": 40181, + "size": 1504987, + "start_time": 3964989369387280, + "stop_time": 3964989469387180, + "real_time": 99999900, + "shape": [ + 4148, + 4362 + ], + "type": "uint32", + "encoding": "bs32-lz4<", + "hash": "3aca01d1417576a9c8c26343a93b29a1" +} \ No newline at end of file diff --git a/python/tests/input/14.json b/python/tests/input/14.json new file mode 100644 index 0000000..e07ecc6 --- /dev/null +++ b/python/tests/input/14.json @@ -0,0 +1,9 @@ +{ + "plugin": "Acquisition", + "parameter": "writeframe", + "type": "string", + "header": { + "acqID": "test_0_0", + "rank": 0 + } +} \ No newline at end of file diff --git a/python/tests/input/15.json b/python/tests/input/15.json new file mode 100644 index 0000000..90de12a --- /dev/null +++ b/python/tests/input/15.json @@ -0,0 +1,7 @@ +{ + "frame": 0, + "offset": 0, + "proc": 4, + "write_duration": 3550, + "flush_duration": 60 +} \ No newline at end of file diff --git a/python/tests/input/16.json b/python/tests/input/16.json new file mode 100644 index 0000000..3328c0e --- /dev/null +++ b/python/tests/input/16.json @@ -0,0 +1,9 @@ +{ + "plugin": "Acquisition", + "parameter": "closefile", + "type": "string", + "header": { + "acqID": "test_0_0", + "rank": 0 + } +} \ No newline at end of file diff --git a/python/tests/input/17.json b/python/tests/input/17.json new file mode 100644 index 0000000..0587d35 --- /dev/null +++ b/python/tests/input/17.json @@ -0,0 +1,4 @@ +{ + "filePath": "/dls/i03/data/2024/cm37235-3/tmp/gary/0/test_0_0_000001.h5", + "close_duration": 263 +} \ No newline at end of file diff --git a/python/tests/input/18.json b/python/tests/input/18.json new file mode 100644 index 0000000..f9cb498 --- /dev/null +++ b/python/tests/input/18.json @@ -0,0 +1,9 @@ +{ + "plugin": "Acquisition", + "parameter": "stopacquisition", + "type": "string", + "header": { + "acqID": "test_0_0", + "rank": 0 + } +} \ No newline at end of file diff --git a/python/tests/input/19 b/python/tests/input/19 new file mode 100644 index 0000000..e69de29 diff --git a/python/tests/input/20.json b/python/tests/input/20.json new file mode 100644 index 0000000..3e81e56 --- /dev/null +++ b/python/tests/input/20.json @@ -0,0 +1,9 @@ +{ + "plugin": "eiger", + "parameter": "eiger-end", + "type": "string", + "header": { + "acqID": "test_0_0", + "series": 40181 + } +} \ No newline at end of file diff --git a/python/tests/input/21 b/python/tests/input/21 new file mode 100644 index 0000000..e69de29 diff --git a/python/tests/test_eiger_meta_writer.py b/python/tests/test_eiger_meta_writer.py index c4ade82..07face9 100644 --- a/python/tests/test_eiger_meta_writer.py +++ b/python/tests/test_eiger_meta_writer.py @@ -1,9 +1,38 @@ +import json +from pathlib import Path + +import h5py as h5 from eiger_detector.data.eiger_meta_writer import EigerMetaWriter from odin_data.meta_writer.meta_writer import MetaWriterConfig +HERE = Path(__file__).parent + + +def test_file_write(tmp_path): -def test_eiger_meta_writer(): writer = EigerMetaWriter( - "writer", "/tmp", [], MetaWriterConfig(sensor_shape=(4362, 4148)) + "test", tmp_path.as_posix(), [], MetaWriterConfig(sensor_shape=(4362, 4148)) ) - assert writer._name == "writer" + + writer._processes_running = [True] + writer._endpoints = ["tcp://127.0.0.1:10008"] + + stream_files = list((HERE / "input").iterdir()) + + # Every other file is header and then data + for header_path, data_path in zip(stream_files[0::2], stream_files[1::2]): + with header_path.open("r") as f: + header = json.load(f) + header["header"]["_endpoint"] = "tcp://127.0.0.1:10008" + + if data_path.suffix == ".json": + with data_path.open("r") as f: + data = json.load(f) + else: + with data_path.open("rb") as f: + data = f.read() + + writer.process_message(header, data) + + with h5.File(tmp_path / "test_meta.h5") as f: + assert f["series"][0] == 40181