From 017f3c2424661e799ea0d2f82ebbb55fe92dcd9c Mon Sep 17 00:00:00 2001 From: Arved Wintzer Date: Wed, 16 Nov 2022 11:44:14 +0100 Subject: [PATCH 1/2] add parent nomad eln file --- multilog/devices/basler_camera.py | 4 +- multilog/devices/daq6510.py | 6 +- multilog/devices/eurotherm.py | 6 +- multilog/devices/ifm_flowmeter.py | 6 +- multilog/devices/optris_ip640.py | 4 +- multilog/devices/process_condition_logger.py | 6 +- multilog/devices/pyrometer_array_lumasense.py | 6 +- multilog/devices/pyrometer_lumasense.py | 6 +- multilog/main.py | 59 ++++++ multilog/nomad/archive_template_Camera.yml | 3 + multilog/nomad/archive_template_IR-Camera.yml | 3 + multilog/nomad/archive_template_main.yml | 182 ++++++++++++++++++ ...mplate.yml => archive_template_sensor.yml} | 3 + 13 files changed, 272 insertions(+), 22 deletions(-) create mode 100644 multilog/nomad/archive_template_main.yml rename multilog/nomad/{archive_template.yml => archive_template_sensor.yml} (93%) diff --git a/multilog/devices/basler_camera.py b/multilog/devices/basler_camera.py index 4906d5d..3d30b0c 100644 --- a/multilog/devices/basler_camera.py +++ b/multilog/devices/basler_camera.py @@ -103,9 +103,9 @@ def init_output(self, directory="./"): f.write("time_abs,time_rel,img-name,\n") with open(f"{self.directory}/device.txt", "w", encoding="utf-8") as f: f.write(self.device_name) - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: diff --git a/multilog/devices/daq6510.py b/multilog/devices/daq6510.py index 46f0d58..a034bef 100644 --- a/multilog/devices/daq6510.py +++ b/multilog/devices/daq6510.py @@ -264,15 +264,15 @@ def init_output(self, directory="./"): with open(self.filename, "w", encoding="utf-8") as f: f.write(units) f.write(header) - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: directory (str, optional): Output directory. Defaults to "./". """ - with open("./multilog/nomad/archive_template.yml") as f: + with open("./multilog/nomad/archive_template_sensor.yml") as f: nomad_template = yaml.safe_load(f) definitions = nomad_template.pop("definitions") data = nomad_template.pop("data") diff --git a/multilog/devices/eurotherm.py b/multilog/devices/eurotherm.py index a871f19..d56fdee 100644 --- a/multilog/devices/eurotherm.py +++ b/multilog/devices/eurotherm.py @@ -91,15 +91,15 @@ def init_output(self, directory="./"): with open(self.filename, "w", encoding="utf-8") as f: f.write(units) f.write(header) - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: directory (str, optional): Output directory. Defaults to "./". """ - with open("./multilog/nomad/archive_template.yml") as f: + with open("./multilog/nomad/archive_template_sensor.yml") as f: nomad_template = yaml.safe_load(f) definitions = nomad_template.pop("definitions") data = nomad_template.pop("data") diff --git a/multilog/devices/ifm_flowmeter.py b/multilog/devices/ifm_flowmeter.py index f64f7f2..1366e7c 100644 --- a/multilog/devices/ifm_flowmeter.py +++ b/multilog/devices/ifm_flowmeter.py @@ -167,15 +167,15 @@ def init_output(self, directory="./"): with open(self.filename, "w", encoding="utf-8") as f: f.write(units) f.write(header) - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: directory (str, optional): Output directory. Defaults to "./". """ - with open("./multilog/nomad/archive_template.yml") as f: + with open("./multilog/nomad/archive_template_sensor.yml") as f: nomad_template = yaml.safe_load(f) definitions = nomad_template.pop("definitions") data = nomad_template.pop("data") diff --git a/multilog/devices/optris_ip640.py b/multilog/devices/optris_ip640.py index dfe1e25..f9aca65 100644 --- a/multilog/devices/optris_ip640.py +++ b/multilog/devices/optris_ip640.py @@ -136,9 +136,9 @@ def init_output(self, directory="./"): with open(f"{self.directory}/_images.csv", "w", encoding="utf-8") as f: f.write("# datetime,s,filename,\n") f.write("time_abs,time_rel,img-name,\n") - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: diff --git a/multilog/devices/process_condition_logger.py b/multilog/devices/process_condition_logger.py index 33c2217..6b9b993 100644 --- a/multilog/devices/process_condition_logger.py +++ b/multilog/devices/process_condition_logger.py @@ -73,15 +73,15 @@ def init_output(self, directory="./"): f"- {condition}: {self.meas_data[condition]} {self.condition_units[condition]}\n" ) f.write("\n## Process condition log\n\n") - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: directory (str, optional): Output directory. Defaults to "./". """ - with open("./multilog/nomad/archive_template.yml") as f: + with open("./multilog/nomad/archive_template_sensor.yml") as f: nomad_template = yaml.safe_load(f) definitions = nomad_template.pop("definitions") data = nomad_template.pop("data") diff --git a/multilog/devices/pyrometer_array_lumasense.py b/multilog/devices/pyrometer_array_lumasense.py index 95724c7..7bfa95c 100644 --- a/multilog/devices/pyrometer_array_lumasense.py +++ b/multilog/devices/pyrometer_array_lumasense.py @@ -130,15 +130,15 @@ def init_output(self, directory="./"): with open(self.filename, "w", encoding="utf-8") as f: f.write(units) f.write(header) - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: directory (str, optional): Output directory. Defaults to "./". """ - with open("./multilog/nomad/archive_template.yml") as f: + with open("./multilog/nomad/archive_template_sensor.yml") as f: nomad_template = yaml.safe_load(f) definitions = nomad_template.pop("definitions") data = nomad_template.pop("data") diff --git a/multilog/devices/pyrometer_lumasense.py b/multilog/devices/pyrometer_lumasense.py index 4127660..48554a3 100644 --- a/multilog/devices/pyrometer_lumasense.py +++ b/multilog/devices/pyrometer_lumasense.py @@ -149,15 +149,15 @@ def init_output(self, directory="./"): with open(self.filename, "w", encoding="utf-8") as f: f.write(units) f.write(header) - self.write_nomad_files(directory) + self.write_nomad_file(directory) - def write_nomad_files(self, directory="./"): + def write_nomad_file(self, directory="./"): """Write .archive.yaml file based on device configuration. Args: directory (str, optional): Output directory. Defaults to "./". """ - with open("./multilog/nomad/archive_template.yml") as f: + with open("./multilog/nomad/archive_template_sensor.yml") as f: nomad_template = yaml.safe_load(f) definitions = nomad_template.pop("definitions") data = nomad_template.pop("data") diff --git a/multilog/main.py b/multilog/main.py index c3d8022..5019693 100644 --- a/multilog/main.py +++ b/multilog/main.py @@ -1,6 +1,7 @@ """This module contains the controller-part of multilog. It sets up the communication between device and visualization and manages the sampling loop.""" +from copy import deepcopy import shutil from PyQt5.QtWidgets import QApplication from PyQt5.QtCore import QTimer, QThread, QObject, pyqtSignal @@ -293,6 +294,7 @@ def init_output_files(self): break for device in self.devices: self.devices[device].init_output(self.directory) + self.write_nomad_file() self.write_metadata() shutil.copy( "./multilog/nomad/base_classes.schema.archive.yaml", @@ -300,6 +302,63 @@ def init_output_files(self): ) self.main_window.set_output_directory(self.directory) + def write_nomad_file(self): + """Write main multilog.archive.yaml including an overview of all devices.""" + with open("./multilog/nomad/archive_template_main.yml") as f: + nomad_dict = yaml.safe_load(f) + data = nomad_dict.pop("data") + multilog_version = ( + subprocess.check_output( + ["git", "describe", "--tags", "--dirty", "--always"] + ) + .strip() + .decode("utf-8") + ) + data["data_processing"].update( + { + "software": f"multilog {multilog_version}", + "sampling_time": self.config["settings"]["dt-main"], + } + ) + + inst_ir_cam = nomad_dict.pop("instrumentation_IR-camera_template") + inst_camera = nomad_dict.pop("instrumentation_camera_template") + inst_sensors = nomad_dict.pop("instrumentation_sensors_template") + for device_name in self.devices: + nomad_name = device_name.replace(" ", "_").replace("-", "_") + if "Optris-IP-640" in device_name: + instrument = deepcopy(inst_ir_cam) + instrument["section"]["quantities"]["ir_camera"]["type"] = f"../upload/raw/{device_name}.archive.yaml#IR_camera" + nomad_dict["definitions"]["sections"]["MeltCzochralski"]["sub_sections"]["instrumentation"]["section"]["sub_sections"].update( + {nomad_name: instrument} + ) + data["instrumentation"][nomad_name] = { + "ir_camera": f"../upload/raw/{device_name}.archive.yaml#data" + } + elif "Basler" in device_name: + instrument = deepcopy(inst_camera) + instrument["section"]["quantities"]["camera"]["type"] = f"../upload/raw/{device_name}.archive.yaml#camera" + nomad_dict["definitions"]["sections"]["MeltCzochralski"]["sub_sections"]["instrumentation"]["section"]["sub_sections"].update( + {nomad_name: instrument} + ) + data["instrumentation"][nomad_name] = { + "camera": f"../upload/raw/{device_name}.archive.yaml#data" + } + else: + instrument = deepcopy(inst_sensors) + instrument["section"]["quantities"]["sensors_list"]["type"] = f"../upload/raw/{device_name}.archive.yaml#Sensors_list" + nomad_dict["definitions"]["sections"]["MeltCzochralski"]["sub_sections"]["instrumentation"]["section"]["sub_sections"].update( + {nomad_name: instrument} + ) + data["instrumentation"][nomad_name] = { + "sensors_list": f"../upload/raw/{device_name}.archive.yaml#data" + } + + nomad_dict.update({"data": data}) + with open(f"{self.directory}/multilog_eln.archive.yaml", "w", encoding="utf-8") as f: + yaml.safe_dump(nomad_dict, f, sort_keys=False) + + def write_metadata(self): """Write a csv file with information about multilog version, python version and operating system. diff --git a/multilog/nomad/archive_template_Camera.yml b/multilog/nomad/archive_template_Camera.yml index 4b340df..2dbcbfd 100644 --- a/multilog/nomad/archive_template_Camera.yml +++ b/multilog/nomad/archive_template_Camera.yml @@ -1,3 +1,6 @@ +# NOMAD template for optical camera, based on +# https://github.com/FAIRmat-NFDI/AreaA-data_modeling_and_schemas/blob/main/melt_czochralski_Dadzis/IRcameras.archive.yaml + definitions: name: 'Optical camera data' sections: diff --git a/multilog/nomad/archive_template_IR-Camera.yml b/multilog/nomad/archive_template_IR-Camera.yml index 7d85b91..5180e80 100644 --- a/multilog/nomad/archive_template_IR-Camera.yml +++ b/multilog/nomad/archive_template_IR-Camera.yml @@ -1,3 +1,6 @@ +# NOMAD template for IR camera, based on +# https://github.com/FAIRmat-NFDI/AreaA-data_modeling_and_schemas/blob/main/melt_czochralski_Dadzis/IRcameras.archive.yaml + definitions: name: 'IR camera data' sections: diff --git a/multilog/nomad/archive_template_main.yml b/multilog/nomad/archive_template_main.yml new file mode 100644 index 0000000..54acd74 --- /dev/null +++ b/multilog/nomad/archive_template_main.yml @@ -0,0 +1,182 @@ +# NOMAD template with main information about the process, based on +# https://github.com/FAIRmat-NFDI/AreaA-data_modeling_and_schemas/blob/main/melt_czochralski_Dadzis/melt_czochralski.schema.archive.yaml +# https://github.com/FAIRmat-NFDI/AreaA-data_modeling_and_schemas/blob/main/melt_czochralski_Dadzis/melt_czochralski.data.archive.yaml + +definitions: + name: 'Electronic Lab Notebook' + sections: # Schemas consist of section definitions + MeltCzochralski: + # Inheriting it from EntryData, makes this section a possible + # EntryArchive.data sub-section + base_sections: + - nomad.datamodel.metainfo.eln.ElnBaseSection + - nomad.datamodel.data.EntryData + m_annotations: + eln: + hide: [name, description] + lane_width: 200px + sub_sections: + users: + section: + quantities: + user: + type: User + shape: ['*'] + m_annotations: + eln: + component: AuthorEditQuantity + process: + section: + base_section: ../upload/raw/base_classes.schema.archive.yaml#Process + sub_sections: + procedure: + repeats: true + section: + base_section: ../upload/raw/base_classes.schema.archive.yaml#Procedure_step + quantities: + crystal_diameter: + type: str + m_annotations: + eln: + component: StringEditQuantity + # fan: + # type: str + # m_annotations: + # eln: + # component: StringEditQuantity + # pump: + # type: str + # m_annotations: + # eln: + # component: StringEditQuantity + # sensors: + # type: str + # m_annotations: + # eln: + # component: StringEditQuantity + data_processing: + section: + quantities: + software: + type: str + m_annotations: + eln: + component: StringEditQuantity + description: "Software used for logging" + sampling_time: + type: np.float64 + unit: ms + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: ms + description: "Time between sampled points" + instrumentation: + section: + m_annotations: + eln: + lane_width: 200px + sub_sections: + furnace: + section: + m_annotations: + eln: + lane_width: 200px + quantities: + furnace_type: + type: str + description: "Furnace description" + m_annotations: + eln: + component: StringEditQuantity + heating: + section: + m_annotations: + eln: + lane_width: 200px + quantities: + heater_type: + type: + type_kind: Enum + type_data: + - Resistance + - Inductor + m_annotations: + eln: + component: EnumEditQuantity + heater_id: + type: str + description: "Inductor or resistance heater name or ID" + m_annotations: + eln: + component: StringEditQuantity + # templates form below will be inserted here + +instrumentation_IR-camera_template: + section: + m_annotations: + eln: + lane_width: 200px + quantities: + ir_camera: + type: ../upload/raw/IRcameras.archive.yaml#IR_camera # this will be replaced +instrumentation_camera_template: + section: + m_annotations: + eln: + lane_width: 200px + quantities: + camera: + type: ../upload/raw/IRcameras.archive.yaml#IR_camera # this will be replaced +instrumentation_sensors_template: + section: + m_annotations: + eln: + lane_width: 200px + # plot: + # title: Sensors for growth monitoring + # x: time + # y: + # - TE_1_front + # - TE_2_vessel + quantities: + sensors_locations: + type: str + description: Picture of sensor locations + m_annotations: + browser: + adaptor: RawFileAdaptor # Allows to navigate to files in the data browser + eln: + component: FileEditQuantity + sensors_list: + type: ../upload/raw/Sensors.archive.yaml#Sensors_list # this will be replaced + #shape: ['*'] + +data: + m_def: MeltCzochralski + data_processing: #section name - not to be filled + software: multilog vXXX # will be set automatically + sampling_time: -1 # will be set automatically + instrumentation: #section name - not to be filled + furnace: #section name - not to be filled + furnace_type: Test-Cz + # heating: #section name - not to be filled + # heater_type: + # heater_id: vXXX + + # the cameras, sensors will be added here automatically + # examples: + # Optris-IP-640: # TODO fill name here + # ir_camera: ../upload/raw/IRcameras.archive.yaml#data # TODO fill name here + # sensors: # TODO fill name here + # sensors_locations: sensor_geometry.png # TODO fill name here + # sensors_list: ../upload/raw/Sensors.archive.yaml#data # TODO fill name here + + # TODO complete this + # process: #section name - not to be filled + # activity_identifier: my_id + # activity_category: crystal growth synthesis + # activity_method: Melt Czochralski + # location: Berlin + # start_time: 2021-03-04 10:15:00+01 + # end_time: 2021-03-04 10:15:00+01 diff --git a/multilog/nomad/archive_template.yml b/multilog/nomad/archive_template_sensor.yml similarity index 93% rename from multilog/nomad/archive_template.yml rename to multilog/nomad/archive_template_sensor.yml index be664bb..c943655 100644 --- a/multilog/nomad/archive_template.yml +++ b/multilog/nomad/archive_template_sensor.yml @@ -1,3 +1,6 @@ +# NOMAD template for measurement devices with sensors recording scalars +# Based on https://github.com/FAIRmat-NFDI/AreaA-data_modeling_and_schemas/blob/main/melt_czochralski_Dadzis/Sensors.archive.yaml + definitions: name: 'Sensors' sections: From e721842d5fe9e3dff108a2139d0b57015fb40196 Mon Sep 17 00:00:00 2001 From: Arved Wintzer Date: Wed, 16 Nov 2022 12:32:48 +0100 Subject: [PATCH 2/2] clean up nomad stuff --- multilog/main.py | 14 ++-- multilog/nomad/archive_template_main.yml | 93 ++++++++++++------------ 2 files changed, 55 insertions(+), 52 deletions(-) diff --git a/multilog/main.py b/multilog/main.py index 5019693..ef5f793 100644 --- a/multilog/main.py +++ b/multilog/main.py @@ -314,20 +314,22 @@ def write_nomad_file(self): .strip() .decode("utf-8") ) + data["process"]["start_time"] = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat(timespec='milliseconds').replace('T', ' ') data["data_processing"].update( { "software": f"multilog {multilog_version}", "sampling_time": self.config["settings"]["dt-main"], + "image_time": self.config["settings"]["dt-camera"], } ) - inst_ir_cam = nomad_dict.pop("instrumentation_IR-camera_template") - inst_camera = nomad_dict.pop("instrumentation_camera_template") - inst_sensors = nomad_dict.pop("instrumentation_sensors_template") + template_ir_cam = nomad_dict.pop("instrumentation_IR-camera_template") + template_camera = nomad_dict.pop("instrumentation_camera_template") + template_sensor = nomad_dict.pop("instrumentation_sensors_template") for device_name in self.devices: nomad_name = device_name.replace(" ", "_").replace("-", "_") if "Optris-IP-640" in device_name: - instrument = deepcopy(inst_ir_cam) + instrument = deepcopy(template_ir_cam) instrument["section"]["quantities"]["ir_camera"]["type"] = f"../upload/raw/{device_name}.archive.yaml#IR_camera" nomad_dict["definitions"]["sections"]["MeltCzochralski"]["sub_sections"]["instrumentation"]["section"]["sub_sections"].update( {nomad_name: instrument} @@ -336,7 +338,7 @@ def write_nomad_file(self): "ir_camera": f"../upload/raw/{device_name}.archive.yaml#data" } elif "Basler" in device_name: - instrument = deepcopy(inst_camera) + instrument = deepcopy(template_camera) instrument["section"]["quantities"]["camera"]["type"] = f"../upload/raw/{device_name}.archive.yaml#camera" nomad_dict["definitions"]["sections"]["MeltCzochralski"]["sub_sections"]["instrumentation"]["section"]["sub_sections"].update( {nomad_name: instrument} @@ -345,7 +347,7 @@ def write_nomad_file(self): "camera": f"../upload/raw/{device_name}.archive.yaml#data" } else: - instrument = deepcopy(inst_sensors) + instrument = deepcopy(template_sensor) instrument["section"]["quantities"]["sensors_list"]["type"] = f"../upload/raw/{device_name}.archive.yaml#Sensors_list" nomad_dict["definitions"]["sections"]["MeltCzochralski"]["sub_sections"]["instrumentation"]["section"]["sub_sections"].update( {nomad_name: instrument} diff --git a/multilog/nomad/archive_template_main.yml b/multilog/nomad/archive_template_main.yml index 54acd74..9d3be39 100644 --- a/multilog/nomad/archive_template_main.yml +++ b/multilog/nomad/archive_template_main.yml @@ -39,39 +39,8 @@ definitions: m_annotations: eln: component: StringEditQuantity - # fan: - # type: str - # m_annotations: - # eln: - # component: StringEditQuantity - # pump: - # type: str - # m_annotations: - # eln: - # component: StringEditQuantity - # sensors: - # type: str - # m_annotations: - # eln: - # component: StringEditQuantity - data_processing: - section: - quantities: - software: - type: str - m_annotations: - eln: - component: StringEditQuantity - description: "Software used for logging" - sampling_time: - type: np.float64 - unit: ms - m_annotations: - eln: - component: NumberEditQuantity - defaultDisplayUnit: ms - description: "Time between sampled points" - instrumentation: + # keep listing other useful parameters here + configuration: section: m_annotations: eln: @@ -110,6 +79,37 @@ definitions: m_annotations: eln: component: StringEditQuantity + data_processing: + section: + quantities: + software: + type: str + m_annotations: + eln: + component: StringEditQuantity + description: "Software used for logging" + sampling_time: + type: np.float64 + unit: ms + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: ms + description: "Time between sampled points" + image_time: + type: np.float64 + unit: ms + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: ms + description: "Time between recorded images" + instrumentation: + section: + m_annotations: + eln: + lane_width: 200px + sub_sections: {} # templates form below will be inserted here instrumentation_IR-camera_template: @@ -154,13 +154,22 @@ instrumentation_sensors_template: data: m_def: MeltCzochralski - data_processing: #section name - not to be filled + process: + # activity_identifier: my_id + # activity_method: Melt Czochralski + start_time: "2021-03-04 10:15:00+01" # will be set automatically + # activity_category: crystal growth synthesis + # end_time: 2021-03-04 10:15:00+01 + # location: Berlin + data_processing: software: multilog vXXX # will be set automatically sampling_time: -1 # will be set automatically - instrumentation: #section name - not to be filled - furnace: #section name - not to be filled - furnace_type: Test-Cz - # heating: #section name - not to be filled + image_time: -1 # will be set automatically + instrumentation: + {} + # furnace: + # furnace_type: Test-Cz + # heating: # heater_type: # heater_id: vXXX @@ -172,11 +181,3 @@ data: # sensors_locations: sensor_geometry.png # TODO fill name here # sensors_list: ../upload/raw/Sensors.archive.yaml#data # TODO fill name here - # TODO complete this - # process: #section name - not to be filled - # activity_identifier: my_id - # activity_category: crystal growth synthesis - # activity_method: Melt Czochralski - # location: Berlin - # start_time: 2021-03-04 10:15:00+01 - # end_time: 2021-03-04 10:15:00+01