Enable pymca_fitter to process hdf5 files. (#8)

With data acquisition planning changes to output XRF measurements as hdf5 files, the pymca fitter service was modified to enable it to fit h5 data files. The PyMCA library on which the fitter service is based has capability to fit data in hdf5 files format, and these changes enable the fitter service to use this capability. Some other changes are also bundled into this PR, which are detailed below:

* Read and process data if supplied as h5 file.
* Refactor internals code to call functions in a more intuitive order.
* Allow h5path and config file path to be specified as custom params to recipe.
* Validate params read from recipe.
* Plot data with matplotlib instead of gnuplot.
* Catch missing fit as an exception.
* Fix bug where the actual beam energy was not being replaced in the config file.
* Fix copying of config file to avoid adding whitespace to the file.
* Populate config file with peaks if not present.
* Write an mca file from the input h5 file.
* Make the cutoff energy used by fitting and plotting consistent.
* Add tests
  * Integration tests for the main run_auto_pymca function using all three accepted data formats.
  * Unit tests for the parse_raw_fluoro function
  *Further unit tests were not added to other functions but should be added as they were considered to be of limited value
* Bump2version 1.0.0.

.github/workflows/tests.yml

@@ -0,0 +1,42 @@
+name: Test CI
+
+on:
+  pull_request:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: |
+            ghcr.io/${{ github.repository }}
+          tags: |
+            type=edge,branch=main
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=sha,prefix=
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: false
+          tags: test
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Test
+        run: |
+          docker run -w /tmp/pymca docker.io/library/test "/opt/pymca/bin/pytest" /tmp/pymca

.gitignore

@@ -1,2 +1,5 @@
 __pycache__
 pymca_zocalo.egg-info/
+.vscode/launch.json
+.vscode/settings.json
+build/*

Dockerfile

@@ -1,6 +1,6 @@
 FROM debian:12-slim
 
-RUN apt-get update && apt-get install -y curl bzip2 gnuplot
+RUN apt-get update && apt-get install -y curl bzip2
 # install miniconda
 RUN curl -LO https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda && \

requirements.txt

@@ -1,5 +1,13 @@
-numpy
-pymca
+bump2version
+h5py
+matplotlib-base=3.8.4
+numpy=1.26.4
+pip
+pymca=5.9.3
+pytest
+pytest-runner
+setuptools
+wheel
 workflows>=1.7
-xraylib
-zocalo
+xraylib=4.1.4
+zocalo

requirements_dev.txt

@@ -1,10 +1,13 @@
 bump2version
+h5py
+matplotlib-base
+numpy
 pip
 pymca
 pytest
 pytest-runner
 setuptools
 wheel
-workflows
+workflows>=1.7
 xraylib
 zocalo

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 1.0.0
 commit = True
 tag = True
 

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="pymca-zocalo",
-    version="0.3.0",
+    version="1.0.0",
     description="PyMca components for automated data processing with Zocalo at Diamond Light Source",
     author="Tom Schoonjans",
     author_email="[email protected]",
@@ -12,12 +12,12 @@
         "License :: OSI Approved :: BSD License",
         "Natural Language :: English",
         "Operating System :: POSIX :: Linux",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.11",
     ],
     packages=find_packages("src"),
     package_dir={"": "src"},
+    include_package_data=True,
+    package_data={"": ["*.cfg"]},
     install_requires=[
         # pymca and xraylib required but listing here causes error when zocalo resolves environment
         # "pymca",

src/pymca_zocalo/__init__.py

@@ -1,3 +1,3 @@
 from .pymca_fitter import DLSPyMcaFitter
 
-__version__ = "0.3.0"
+__version__ = "1.0.0"

src/pymca_zocalo/data/pymca_new.cfg

@@ -0,0 +1,180 @@
+
+[attenuators]
+kapton = 0, -, 0.0, 0.0, 1.0
+atmosphere = 0, -, 0.0, 0.0, 1.0
+Matrix = 0, MULTILAYER, 0.0, 0.0, 45.0, 45.0, 0, 90.0
+deadlayer = 0, Si1, 2.33, 0.002, 1.0
+BeamFilter1 = 0, -, 0.0, 0.0, 1.0
+BeamFilter0 = 0, -, 0.0, 0.0, 1.0
+absorber = 0, -, 0.0, 0.0, 1.0
+window = 0, -, 0.0, 0.0, 1.0
+contact = 0, Au1, 19.37, 1e-06, 1.0
+Filter 6 = 0, -, 0.0, 0.0, 1.0
+Filter 7 = 0, -, 0.0, 0.0, 1.0
+Detector = 0, Si1, 2.33, 0.5, 1.0
+
+[fit]
+stripwidth = 20
+linearfitflag = 0
+xmin = 0
+scatterflag = 1
+snipwidth = 30
+stripfilterwidth = 1
+escapeflag = 1
+exppolorder = 6
+fitweight = 1
+stripflag = 1
+stripanchorsflag = 0
+use_limit = 0
+maxiter = 10
+stripiterations = 20000
+sumflag = 1
+linpolorder = 5
+stripalgorithm = 0
+deltaonepeak = 0.01
+deltachi = 0.001
+continuum = 0
+hypermetflag = 3
+stripconstant = 1.0
+xmax = 4096
+fitfunction = 0
+energy = 20.0, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None
+stripanchorslist = 0, 0, 0, 0
+energyscatter = 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+energyweight = 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
+energyflag = 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+
+[multilayer]
+Layer3 = 0, -, 0.0, 0.0
+Layer2 = 0, -, 0.0, 0.0
+Layer1 = 0, -, 0.0, 0.0
+Layer0 = 0, -, 0.0, 0.0
+Layer7 = 0, -, 0.0, 0.0
+Layer6 = 0, -, 0.0, 0.0
+Layer5 = 0, -, 0.0, 0.0
+Layer4 = 0, -, 0.0, 0.0
+Layer9 = 0, -, 0.0, 0.0
+Layer8 = 0, -, 0.0, 0.0
+
+[tube]
+windowdensity = 1.848
+anodedensity = 10.5
+windowthickness = 0.0125
+anodethickness = 0.0002
+transmission = 0
+alphax = 90.0
+deltaplotting = 0.1
+window = Be
+filter1thickness = 0.0
+anode = Ag
+voltage = 30.0
+filter1density = 0.000118
+alphae = 90.0
+filter1 = He
+
+[materials]
+
+[materials.Kapton]
+Comment = Kapton 100 HN 25 micron density=1.42 g/cm3
+Density = 1.42
+Thickness = 0.0025
+CompoundFraction = 0.628772, 0.066659, 0.304569
+CompoundList = C1, N1, O1
+
+[materials.Teflon]
+Comment = Teflon density=2.2 g/cm3
+Density = 2.2
+CompoundFraction = 0.240183, 0.759817
+CompoundList = C1, F1
+
+[materials.Gold]
+Comment = Gold
+CompoundFraction = 1.0
+Thickness = 1e-06
+Density = 19.37
+CompoundList = Au
+
+[materials.Air]
+Comment = Dry Air (Near sea level) density=0.001204790 g/cm3
+Thickness = 1.0
+Density = 0.0012048
+CompoundFraction = 0.000124, 0.75527, 0.23178, 0.012827, 3.2e-06
+CompoundList = C1, N1, O1, Ar1, Kr1
+
+[materials.Water]
+Comment = Water density=1.0 g/cm3
+CompoundFraction = 1.0
+Density = 1.0
+CompoundList = H2O1
+
+[materials.Viton]
+Comment = Viton Fluoroelastomer density=1.8 g/cm3
+Density = 1.8
+CompoundFraction = 0.009417, 0.280555, 0.710028
+CompoundList = H1, C1, F1
+
+[materials.Goethite]
+Comment = Mineral FeO(OH) density from 3.3 to 4.3 density=4.3 g/cm3
+CompoundFraction = 1.0
+CompoundList = Fe1O2H1
+Thickness = 0.1
+Density = 4.3
+
+[materials.Mylar]
+Comment = Mylar (Polyethylene Terephthalate) density=1.40 g/cm3
+Density = 1.4
+CompoundFraction = 0.041959, 0.625017, 0.333025
+CompoundList = H1, C1, O1
+
+[concentrations]
+usemultilayersecondary = 0
+reference = Auto
+area = 30.0
+flux = 10000000000.0
+time = 1.0
+useattenuators = 1
+usematrix = 0
+mmolarflag = 0
+distance = 10.0
+
+[detector]
+noise = 0.1
+fixednoise = 0
+fixedgain = 0
+deltafano = 0.114
+fixedfano = 0
+sum = 1e-08
+deltasum = 1e-08
+fano = 0.114
+fixedsum = 0
+fixedzero = 0
+zero = 0.00
+deltazero = 0.1
+deltanoise = 0.05
+deltagain = 0.001
+detele = Si
+nthreshold = 4
+gain = 0.01
+
+[peakshape]
+lt_arearatio = 0.02
+fixedlt_arearatio = 0
+fixedeta_factor = 0
+st_arearatio = 0.05
+deltalt_arearatio = 0.015
+deltaeta_factor = 0.02
+deltalt_sloperatio = 7.0
+deltastep_heightratio = 5e-05
+st_sloperatio = 0.5
+lt_sloperatio = 10.0
+fixedlt_sloperatio = 0
+deltast_arearatio = 0.03
+eta_factor = 0.02
+fixedst_sloperatio = 0
+fixedst_arearatio = 0
+deltast_sloperatio = 0.49
+step_heightratio = 0.0001
+fixedstep_heightratio = 0
+
+[peaks]
+