From 733ef8d357c8f2d45d842dce822b8e42673bce2b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 10 Aug 2023 05:14:35 -0700 Subject: [PATCH 01/13] Python: 3.8+ (#1502) Python 3.7 went EOL last month. This bumps our supported versions to 3.8+. --- CMakeLists.txt | 2 +- Dockerfile | 28 ++-------------------------- README.md | 2 +- conda.yml | 2 +- docs/source/dev/dependencies.rst | 2 +- setup.py | 3 +-- 6 files changed, 7 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c0416d5fa9..d1c0af5d91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -397,7 +397,7 @@ if(CMAKE_VERSION VERSION_LESS 3.18.0) set(_PY_DEV_MODULE Development) endif() if(openPMD_USE_PYTHON STREQUAL AUTO) - find_package(Python 3.7.0 COMPONENTS Interpreter ${_PY_DEV_MODULE}) + find_package(Python 3.8.0 COMPONENTS Interpreter ${_PY_DEV_MODULE}) if(Python_FOUND) if(openPMD_USE_INTERNAL_PYBIND11) add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11") diff --git a/Dockerfile b/Dockerfile index 54e9050987..5f08b571c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,8 @@ FROM quay.io/pypa/manylinux2010_x86_64 as build-env # FROM quay.io/pypa/manylinux1_x86_64 as build-env ENV DEBIAN_FRONTEND noninteractive -# Python 3.7-3.11 via "37m 38 39 311" -ARG PY_VERSIONS="37m 38 39 310 311" +# Python 3.8-3.11 via "38 39 311" +ARG PY_VERSIONS="38 39 310 311" # static libs need relocatable symbols for linking to shared python lib ENV CFLAGS="-fPIC ${CFLAGS}" @@ -112,30 +112,6 @@ RUN for whl in /opt/src/dist/*.whl; do \ && du -hs /opt/src/dist/* \ && du -hs /wheelhouse/* -# test in fresh env: Debian:Buster + Python 3.7 -FROM debian:buster -ENV DEBIAN_FRONTEND noninteractive -COPY --from=build-env /wheelhouse/openPMD_api-*-cp37-cp37m-manylinux2010_x86_64.whl . -RUN apt-get update \ - && apt-get install -y --no-install-recommends python3 python3-pip \ - && rm -rf /var/lib/apt/lists/* - # binutils -RUN python3 --version \ - && python3 -m pip install -U pip \ - && python3 -m pip install openPMD_api-*-cp37-cp37m-manylinux2010_x86_64.whl -RUN find / -name "openpmd*" -RUN ls -hal /usr/local/lib/python3.7/dist-packages/ -RUN ls -hal /usr/local/lib/python3.7/dist-packages/openpmd_api/ -# RUN ls -hal /usr/local/lib/python3.7/dist-packages/.libsopenpmd_api -# RUN objdump -x /usr/local/lib/python3.7/dist-packages/openpmd_api.cpython-37m-x86_64-linux-gnu.so | grep RPATH -RUN ldd /usr/local/lib/python3.7/dist-packages/openpmd_api/openpmd_api_cxx.cpython-37m-x86_64-linux-gnu.so -RUN python3 -c "import openpmd_api as io; print(io.__version__); print(io.variants)" -RUN python3 -m openpmd_api.ls --help -RUN openpmd-ls --help -#RUN echo "* soft core 100000" >> /etc/security/limits.conf && \ -# python3 -c "import openpmd_api as io"; \ -# gdb -ex bt -c core - # test in fresh env: Debian:Sid + Python 3.8 FROM debian:sid ENV DEBIAN_FRONTEND noninteractive diff --git a/README.md b/README.md index afb44932eb..a2983def96 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ while those can be built either with or without: Optional language bindings: * Python: - * Python 3.7 - 3.11 + * Python 3.8 - 3.11 * pybind11 2.11.1+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/conda.yml b/conda.yml index 2bb1713efd..e82567b2ae 100644 --- a/conda.yml +++ b/conda.yml @@ -36,7 +36,7 @@ dependencies: - pre-commit - pyarrow # for dask # - pybind11 # shipped internally - - python>=3.7 + - python>=3.8 # just a note for later hackery, we could install pip packages inside the env, too: # - pip: diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index e82a330c8a..cc66b85587 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -39,7 +39,7 @@ Optional: language bindings * Python: - * Python 3.7 - 3.11 + * Python 3.8 - 3.11 * pybind11 2.11.1+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/setup.py b/setup.py index 9db688042a..8d3a7e3348 100644 --- a/setup.py +++ b/setup.py @@ -192,7 +192,7 @@ def build_extension(self, ext): cmdclass=dict(build_ext=CMakeBuild), # scripts=['openpmd-ls'], zip_safe=False, - python_requires='>=3.7', + python_requires='>=3.8', # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli @@ -221,7 +221,6 @@ def build_extension(self, ext): 'Topic :: Database :: Front-Ends', 'Programming Language :: C++', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', From 0387b6f7ee6a8a7f404b209cf58a1a7997b52f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 11 Aug 2023 00:29:31 +0200 Subject: [PATCH 02/13] Optional debugging output for AbstractIOHandlerImpl::flush() (#1495) * Optional debugging output for AbstractIOHandlerImpl::flush() * Add an environment variable for this --- docs/source/usage/workflow.rst | 6 + include/openPMD/IO/AbstractIOHandlerImpl.hpp | 197 +---------- src/IO/AbstractIOHandlerImpl.cpp | 351 +++++++++++++++++++ 3 files changed, 364 insertions(+), 190 deletions(-) diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index 843bd2d76d..61ef593a2e 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -98,3 +98,9 @@ Attributes are (currently) unaffected by this: Some backends (e.g. the BP5 engine of ADIOS2) have multiple implementations for the openPMD-api-level guarantees of flush points. For user-guided selection of such implementations, ``Series::flush`` and ``Attributable::seriesFlush()`` take an optional JSON/TOML string as a parameter. See the section on :ref:`backend-specific configuration ` for details. + +Deferred Data API Contract +-------------------------- + +A verbose debug log can optionally be printed to the standard error output by specifying the environment variable ``OPENPMD_VERBOSE=1``. +Note that this functionality is at the current time still relatively basic. diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 1bca04f5d3..8d13f3feb8 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -26,7 +26,6 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include -#include namespace openPMD { @@ -36,198 +35,11 @@ class Writable; class AbstractIOHandlerImpl { public: - AbstractIOHandlerImpl(AbstractIOHandler *handler) : m_handler{handler} - {} + AbstractIOHandlerImpl(AbstractIOHandler *handler); virtual ~AbstractIOHandlerImpl() = default; - std::future flush() - { - using namespace auxiliary; - - while (!(*m_handler).m_work.empty()) - { - IOTask &i = (*m_handler).m_work.front(); - try - { - switch (i.operation) - { - using O = Operation; - case O::CREATE_FILE: - createFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CHECK_FILE: - checkFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CREATE_PATH: - createPath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CREATE_DATASET: - createDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::EXTEND_DATASET: - extendDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::OPEN_FILE: - openFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CLOSE_FILE: - closeFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::OPEN_PATH: - openPath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CLOSE_PATH: - closePath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::OPEN_DATASET: - openDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_FILE: - deleteFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_PATH: - deletePath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_DATASET: - deleteDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_ATT: - deleteAttribute( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::WRITE_DATASET: - writeDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::WRITE_ATT: - writeAttribute( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::READ_DATASET: - readDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::GET_BUFFER_VIEW: - getBufferView( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::READ_ATT: - readAttribute( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::LIST_PATHS: - listPaths( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::LIST_DATASETS: - listDatasets( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::LIST_ATTS: - listAttributes( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::ADVANCE: - advance( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::AVAILABLE_CHUNKS: - availableChunks( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::KEEP_SYNCHRONOUS: - keepSynchronous( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DEREGISTER: - deregister( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - } - } - catch (...) - { - std::cerr << "[AbstractIOHandlerImpl] IO Task " - << internal::operationAsString(i.operation) - << " failed with exception. Clearing IO queue and " - "passing on the exception." - << std::endl; - while (!m_handler->m_work.empty()) - { - m_handler->m_work.pop(); - } - throw; - } - (*m_handler).m_work.pop(); - } - return std::future(); - } + std::future flush(); /** * Close the file corresponding with the writable and release file handles. @@ -592,5 +404,10 @@ class AbstractIOHandlerImpl deregister(Writable *, Parameter const ¶m) = 0; AbstractIOHandler *m_handler; + bool m_verboseIOTasks = false; + + // Args will be forwarded to std::cerr if m_verboseIOTasks is true + template + void writeToStderr(Args &&...) const; }; // AbstractIOHandlerImpl } // namespace openPMD diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index d762df6a1f..af827704a1 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -20,14 +20,365 @@ */ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" + +#include "openPMD/auxiliary/Environment.hpp" #include "openPMD/backend/Writable.hpp" +#include + namespace openPMD { +AbstractIOHandlerImpl::AbstractIOHandlerImpl(AbstractIOHandler *handler) + : m_handler{handler} +{ + if (auxiliary::getEnvNum("OPENPMD_VERBOSE", 0) != 0) + { + m_verboseIOTasks = true; + } +} + void AbstractIOHandlerImpl::keepSynchronous( Writable *writable, Parameter param) { writable->abstractFilePosition = param.otherWritable->abstractFilePosition; writable->written = true; } + +template +void AbstractIOHandlerImpl::writeToStderr([[maybe_unused]] Args &&...args) const +{ + if (m_verboseIOTasks) + { + (std::cerr << ... << args) << std::endl; + } +} + +std::future AbstractIOHandlerImpl::flush() +{ + using namespace auxiliary; + + while (!(*m_handler).m_work.empty()) + { + IOTask &i = (*m_handler).m_work.front(); + try + { + switch (i.operation) + { + using O = Operation; + case O::CREATE_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CREATE_FILE: ", + parameter.name); + createFile(i.writable, parameter); + break; + } + case O::CHECK_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CHECK_FILE: ", + parameter.name); + checkFile(i.writable, parameter); + break; + } + case O::CREATE_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CREATE_PATH: ", + parameter.path); + createPath(i.writable, parameter); + break; + } + case O::CREATE_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CREATE_DATASET: ", + parameter.name); + createDataset(i.writable, parameter); + break; + } + case O::EXTEND_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] EXTEND_DATASET"); + extendDataset(i.writable, parameter); + break; + } + case O::OPEN_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] OPEN_FILE: ", + parameter.name); + openFile(i.writable, parameter); + break; + } + case O::CLOSE_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] CLOSE_FILE"); + closeFile(i.writable, parameter); + break; + } + case O::OPEN_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] OPEN_PATH: ", + parameter.path); + openPath(i.writable, parameter); + break; + } + case O::CLOSE_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] CLOSE_PATH"); + closePath(i.writable, parameter); + break; + } + case O::OPEN_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] OPEN_DATASET: ", + parameter.name); + openDataset(i.writable, parameter); + break; + } + case O::DELETE_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DELETE_FILE"); + deleteFile(i.writable, parameter); + break; + } + case O::DELETE_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DELETE_PATH"); + deletePath(i.writable, parameter); + break; + } + case O::DELETE_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] DELETE_DATASET"); + deleteDataset(i.writable, parameter); + break; + } + case O::DELETE_ATT: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DELETE_ATT"); + deleteAttribute(i.writable, parameter); + break; + } + case O::WRITE_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] WRITE_DATASET"); + writeDataset(i.writable, parameter); + break; + } + case O::WRITE_ATT: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] WRITE_ATT: (", + parameter.dtype, + ") ", + parameter.name); + writeAttribute(i.writable, parameter); + break; + } + case O::READ_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] READ_DATASET"); + readDataset(i.writable, parameter); + break; + } + case O::GET_BUFFER_VIEW: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] GET_BUFFER_VIEW"); + getBufferView(i.writable, parameter); + break; + } + case O::READ_ATT: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] READ_ATT: ", + parameter.name); + readAttribute(i.writable, parameter); + break; + } + case O::LIST_PATHS: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] LIST_PATHS"); + listPaths(i.writable, parameter); + break; + } + case O::LIST_DATASETS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] LIST_DATASETS"); + listDatasets(i.writable, parameter); + break; + } + case O::LIST_ATTS: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] LIST_ATTS"); + listAttributes(i.writable, parameter); + break; + } + case O::ADVANCE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] ADVANCE"); + advance(i.writable, parameter); + break; + } + case O::AVAILABLE_CHUNKS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] AVAILABLE_CHUNKS"); + availableChunks(i.writable, parameter); + break; + } + case O::KEEP_SYNCHRONOUS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] KEEP_SYNCHRONOUS"); + keepSynchronous(i.writable, parameter); + break; + } + case O::DEREGISTER: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DEREGISTER"); + deregister(i.writable, parameter); + break; + } + } + } + catch (...) + { + std::cerr << "[AbstractIOHandlerImpl] IO Task " + << internal::operationAsString(i.operation) + << " failed with exception. Clearing IO queue and " + "passing on the exception." + << std::endl; + while (!m_handler->m_work.empty()) + { + m_handler->m_work.pop(); + } + throw; + } + (*m_handler).m_work.pop(); + } + return std::future(); +} } // namespace openPMD From 64c6b637f26916ceb93a892433bf3e28eda0ff03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:08:36 -0700 Subject: [PATCH 03/13] [pre-commit.ci] pre-commit autoupdate (#1504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/Lucas-C/pre-commit-hooks: v1.5.3 → v1.5.4](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.5.3...v1.5.4) - [github.com/hadialqattan/pycln: v2.2.1 → v2.2.2](https://github.com/hadialqattan/pycln/compare/v2.2.1...v2.2.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d7665da04..4262c09075 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,7 +49,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.3 + rev: v1.5.4 hooks: - id: remove-tabs @@ -80,7 +80,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v2.2.1 + rev: v2.2.2 hooks: - id: pycln name: pycln (python) From 99daca78e9ef2207b78e922ad393cadf9a8ab019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Ordyna?= Date: Thu, 17 Aug 2023 06:21:42 +0200 Subject: [PATCH 04/13] make it possible to manually set chunks when loading dask arrays (#1477) * make it possible to manually set chunks when loading dask arrays * Apply suggestions from code review Copyright and typos in setting chunks for dask arrays. Co-authored-by: Axel Huebl --- docs/source/analysis/dask.rst | 3 + src/binding/python/openpmd_api/DaskArray.py | 102 +++++++++++--------- 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/docs/source/analysis/dask.rst b/docs/source/analysis/dask.rst index e00dce0e98..432bdd5790 100644 --- a/docs/source/analysis/dask.rst +++ b/docs/source/analysis/dask.rst @@ -41,6 +41,9 @@ The central Python API calls to convert to DASK datatypes are the ``ParticleSpec # note: no series.flush() needed +The ``to_dask_array`` method will automatically set Dask array chunking based on the available chunks in the read data set. +The default behavior can be overridden by passing an additional keyword argument ``chunks``, see the `dask.array.from_array documentation `__ for more details. +For example, to chunk only along the outermost axis in a 3D dataset using the default Dask array chunk size, call ``to_dask_array(chunks={0: 'auto', 1: -1, 2: -1})``. Example ------- diff --git a/src/binding/python/openpmd_api/DaskArray.py b/src/binding/python/openpmd_api/DaskArray.py index 8c6fc3001b..709b668be8 100644 --- a/src/binding/python/openpmd_api/DaskArray.py +++ b/src/binding/python/openpmd_api/DaskArray.py @@ -1,8 +1,8 @@ """ This file is part of the openPMD-api. -Copyright 2021 openPMD contributors -Authors: Axel Huebl +Copyright 2021-2023 openPMD contributors +Authors: Axel Huebl, Pawel Ordyna License: LGPLv3+ """ import math @@ -49,7 +49,7 @@ def __getitem__(self, slices): return data -def record_component_to_daskarray(record_component): +def record_component_to_daskarray(record_component, chunks=None): """ Load a RecordComponent into a Dask.array. @@ -57,6 +57,10 @@ def record_component_to_daskarray(record_component): ---------- record_component : openpmd_api.Record_Component A record component class in openPMD-api. + chunks : chunks parameter to pass to dask.array.from_array. + See dask documentation for more details. + When set to None (default) the chunking will be automaticaly + determined based on record_component.available_chunks(). Returns ------- @@ -84,54 +88,56 @@ def record_component_to_daskarray(record_component): if not found_dask: raise ImportError("dask NOT found. Install dask for Dask DataFrame " "support.") - - # get optimal chunks - chunks = record_component.available_chunks() - - # sort and prepare the chunks for Dask's array API - # https://docs.dask.org/en/latest/array-chunks.html - # https://docs.dask.org/en/latest/array-api.html?highlight=from_array#other-functions - # sorted and unique - offsets_per_dim = list(map(list, zip(*[chunk.offset for chunk in chunks]))) - offsets_sorted_unique_per_dim = [sorted(set(o)) for o in offsets_per_dim] - - # print("offsets_sorted_unique_per_dim=", - # list(offsets_sorted_unique_per_dim)) - - # case 1: PIConGPU static load balancing (works with Dask assumptions, - # chunk option no. 3) - # all chunks in the same column have the same column width although - # individual columns have different widths - # case 2: AMReX boxes - # all chunks are multiple of a common block size, offsets are a multiple - # of a common blocksize - # problem: too limited description in Dask - # https://github.com/dask/dask/issues/7475 - # work-around: create smaller chunks (this incurs a read cost) by forcing - # into case 1 - # (this can lead to larger blocks than using the gcd of the - # extents aka AMReX block size) - common_chunk_widths_per_dim = list() - for d, offsets_in_dim in enumerate(offsets_sorted_unique_per_dim): - # print("d=", d, offsets_in_dim, record_component.shape[d]) - offsets_in_dim_arr = np.array(offsets_in_dim) - # note: this is in the right order of rows/columns, contrary to a - # sorted extent list from chunks - extents_in_dim = np.zeros_like(offsets_in_dim_arr) - extents_in_dim[:-1] = offsets_in_dim_arr[1:] - extents_in_dim[-1] = record_component.shape[d] - if len(extents_in_dim) > 1: - extents_in_dim[:-1] -= offsets_in_dim_arr[:-1] - extents_in_dim[-1] -= offsets_in_dim_arr[-1] - # print("extents_in_dim=", extents_in_dim) - common_chunk_widths_per_dim.append(tuple(extents_in_dim)) - - common_chunk_widths_per_dim = tuple(common_chunk_widths_per_dim) - # print("common_chunk_widths_per_dim=", common_chunk_widths_per_dim) + if chunks is None: + # get optimal chunks + chunks = record_component.available_chunks() + + # sort and prepare the chunks for Dask's array API + # https://docs.dask.org/en/latest/array-chunks.html + # https://docs.dask.org/en/latest/array-api.html?highlight=from_array#other-functions + # sorted and unique + offsets_per_dim = list( + map(list, zip(*[chunk.offset for chunk in chunks]))) + offsets_sorted_unique_per_dim = [ + sorted(set(o)) for o in offsets_per_dim] + + # print("offsets_sorted_unique_per_dim=", + # list(offsets_sorted_unique_per_dim)) + + # case 1: PIConGPU static load balancing (works with Dask assumptions, + # chunk option no. 3) + # all chunks in the same column have the same column width although + # individual columns have different widths + # case 2: AMReX boxes + # all chunks are multiple of a common block size, offsets + # are a multiple of a common blocksize + # problem: too limited description in Dask + # https://github.com/dask/dask/issues/7475 + # work-around: create smaller chunks (this incurs a read cost) + # by forcing into case 1 + # (this can lead to larger blocks than using + # the gcd of the extents aka AMReX block size) + common_chunk_widths_per_dim = list() + for d, offsets_in_dim in enumerate(offsets_sorted_unique_per_dim): + # print("d=", d, offsets_in_dim, record_component.shape[d]) + offsets_in_dim_arr = np.array(offsets_in_dim) + # note: this is in the right order of rows/columns, contrary to a + # sorted extent list from chunks + extents_in_dim = np.zeros_like(offsets_in_dim_arr) + extents_in_dim[:-1] = offsets_in_dim_arr[1:] + extents_in_dim[-1] = record_component.shape[d] + if len(extents_in_dim) > 1: + extents_in_dim[:-1] -= offsets_in_dim_arr[:-1] + extents_in_dim[-1] -= offsets_in_dim_arr[-1] + # print("extents_in_dim=", extents_in_dim) + common_chunk_widths_per_dim.append(tuple(extents_in_dim)) + + chunks = tuple(common_chunk_widths_per_dim) + # print("chunks=", chunks) da = from_array( DaskRecordComponent(record_component), - chunks=common_chunk_widths_per_dim, + chunks=chunks, # name=None, asarray=True, fancy=False, From 9ec90b68b91203513046147e1f4336edb8b9547b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 17 Aug 2023 19:34:07 +0200 Subject: [PATCH 05/13] TOML Backend (#1436) * TOML backend * Add documentation for TOML * Fixes for long double and long integer types * Only run TOML tests if TOML is available TOML is not shown as available on NVIDIA compilers * Deactivate long double entirely for JSON/TOML * CI fix: unused variable * Hide/deactivate/warn Toml backend on nvcc compilers https://github.com/ToruNiina/toml11/issues/205 * Usage notes for JSON / TOML * Update comment in test/python/unittest/API/APITest.py Co-authored-by: Axel Huebl * Update documentation text Co-authored-by: Axel Huebl --------- Co-authored-by: Axel Huebl --- docs/source/backends/json.rst | 43 +++- include/openPMD/IO/Format.hpp | 1 + include/openPMD/IO/JSON/JSONIOHandler.hpp | 7 +- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 45 +++- include/openPMD/auxiliary/TypeTraits.hpp | 16 ++ src/Format.cpp | 4 + src/IO/AbstractIOHandlerHelper.cpp | 15 +- src/IO/JSON/JSONIOHandler.cpp | 10 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 225 +++++++++++++----- src/Series.cpp | 3 +- src/auxiliary/JSON.cpp | 8 +- src/config.cpp | 11 + test/CoreTest.cpp | 2 +- test/SerialIOTest.cpp | 57 ++++- test/python/unittest/API/APITest.py | 5 +- 15 files changed, 349 insertions(+), 103 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 752497aa09..48ec6b1f44 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -1,10 +1,19 @@ .. _backends-json: -JSON -==== +JSON/TOML +========= -openPMD supports writing to and reading from JSON files. -The JSON backend is always available. +openPMD supports writing to and reading from JSON and TOML files. +The JSON and TOML backends are always available. + +.. note:: + + Both the JSON and the TOML backends are not intended for large-scale data I/O. + + The JSON backend is mainly intended for prototyping and learning, or similar workflows where setting up a large IO backend such as HDF5 or ADIOS2 is perceived as obstructive. It can also be used for small datasets that need to be stored in text format rather than binary. + + The TOML backend is intended for exchanging the *structure* of a data series without its "heavy" data fields. + For instance, one can easily create and exchange human-readable, machine-actionable data configurations for experiments and simulations. JSON File Format @@ -43,9 +52,17 @@ Every such attribute is itself a JSON object with two keys: * ``datatype``: A string describing the type of the value. * ``value``: The actual value of type ``datatype``. +TOML File Format +---------------- + +A TOML file uses the file ending ``.toml``. The TOML backend is chosen by creating a ``Series`` object with a filename that has this file ending. + +The TOML backend internally works with JSON datasets and converts to/from TOML during I/O. +As a result, data layout and usage are equivalent to the JSON backend. + -Restrictions ------------- +JSON Restrictions +----------------- For creation of JSON serializations (i.e. writing), the restrictions of the JSON backend are equivalent to those of the `JSON library by Niels Lohmann `_ @@ -77,6 +94,20 @@ The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved an A parallel (i.e. MPI) implementation is *not* available. +TOML Restrictions +----------------- + +Note that the JSON datatype-specific restrictions do not automatically hold for TOML, as those affect only the representation on disk, not the internal representation. + +TOML supports most numeric types, with the support for long double and long integer types being platform-defined. +Special floating point values such as NaN are also support. + +TOML does not support null values. + +The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved and must not be used for base/mesh/particles path, records and their components. + +A parallel (i.e. MPI) implementation is *not* available. + Example ------- diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 43ec4d04a1..858da29a40 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -35,6 +35,7 @@ enum class Format ADIOS2_SST, ADIOS2_SSC, JSON, + TOML, DUMMY }; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 0cdc6f3c36..452098137e 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -29,7 +29,12 @@ namespace openPMD class JSONIOHandler : public AbstractIOHandler { public: - JSONIOHandler(std::string path, Access at); + JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON config, + JSONIOHandlerImpl::FileFormat, + std::string originalExtension); ~JSONIOHandler() override; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 7f10f62cd9..c935647665 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -26,8 +26,10 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/IO/JSON/JSONFilePosition.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/config.hpp" +#include #include #include @@ -153,7 +155,17 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl using json = nlohmann::json; public: - explicit JSONIOHandlerImpl(AbstractIOHandler *); + enum class FileFormat + { + Json, + Toml + }; + + explicit JSONIOHandlerImpl( + AbstractIOHandler *, + openPMD::json::TracingJSON config, + FileFormat, + std::string originalExtension); ~JSONIOHandlerImpl() override; @@ -229,15 +241,25 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // files that have logically, but not physically been written to std::unordered_set m_dirty; + /* + * Is set by constructor. + */ + FileFormat m_fileFormat{}; + + std::string m_originalExtension; + // HELPER FUNCTIONS - // will use the IOHandler to retrieve the correct directory - // shared pointer to circumvent the fact that c++ pre 17 does - // not enforce (only allow) copy elision in return statements - std::shared_ptr getFilehandle( - File, - Access access); //, Access - // m_frontendAccess=this->m_handler->m_frontendAccess); + // will use the IOHandler to retrieve the correct directory. + // first tuple element will be the underlying opened file handle. + // if Access is read mode, then the second tuple element will be the istream + // casted to precision std::numeric_limits::digits10 + 1, else null. + // if Access is write mode, then the second tuple element will be the + // ostream casted to precision std::numeric_limits::digits10 + 1, + // else null. first tuple element needs to be a pointer, since the casted + // streams are references only. + std::tuple, std::istream *, std::ostream *> + getFilehandle(File, Access access); // full operating system path of the given file std::string fullPath(File); @@ -272,15 +294,13 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // essentially: m_i = \prod_{j=0}^{i-1} extent_j static Extent getMultiplicators(Extent const &extent); - static nlohmann::json initializeNDArray(Extent const &extent); - static Extent getExtent(nlohmann::json &j); // remove single '/' in the beginning and end of a string static std::string removeSlashes(std::string); template - static bool hasKey(nlohmann::json &, KeyT &&key); + static bool hasKey(nlohmann::json const &, KeyT &&key); // make sure that the given path exists in proper form in // the passed json value @@ -366,7 +386,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl struct AttributeReader { template - static void call(nlohmann::json &, Parameter &); + static void + call(nlohmann::json const &, Parameter &); static constexpr char const *errorMsg = "JSON: writeAttribute"; }; diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 923cfb5be2..3e5a36774e 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -24,6 +24,7 @@ #include "openPMD/auxiliary/UniquePtr.hpp" #include +#include #include // size_t #include #include @@ -56,6 +57,18 @@ namespace detail static constexpr bool value = true; }; + template + struct IsComplex + { + static constexpr bool value = false; + }; + + template + struct IsComplex> + { + static constexpr bool value = true; + }; + template struct IsPointer { @@ -114,6 +127,9 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsContiguousContainer_v = IsVector_v || IsArray_v; +template +inline constexpr bool IsComplex_v = detail::IsComplex::value; + namespace { // see https://en.cppreference.com/w/cpp/language/if diff --git a/src/Format.cpp b/src/Format.cpp index d5a8acf5f3..8a6ead832a 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -43,6 +43,8 @@ Format determineFormat(std::string const &filename) return Format::ADIOS2_SSC; if (auxiliary::ends_with(filename, ".json")) return Format::JSON; + if (auxiliary::ends_with(filename, ".toml")) + return Format::TOML; // Format might still be specified via JSON return Format::DUMMY; @@ -66,6 +68,8 @@ std::string suffix(Format f) return ".ssc"; case Format::JSON: return ".json"; + case Format::TOML: + return ".toml"; default: return ""; } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 4cd74a4de2..c6cd69f8a5 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -194,7 +194,20 @@ std::unique_ptr createIOHandler( std::move(originalExtension)); case Format::JSON: return constructIOHandler( - "JSON", path, access); + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Json, + std::move(originalExtension)); + case Format::TOML: + return constructIOHandler( + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Toml, + std::move(originalExtension)); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending? Specified " diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 10ffce9927..7eb8a57278 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -25,8 +25,14 @@ namespace openPMD { JSONIOHandler::~JSONIOHandler() = default; -JSONIOHandler::JSONIOHandler(std::string path, Access at) - : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} +JSONIOHandler::JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON jsonCfg, + JSONIOHandlerImpl::FileFormat format, + std::string originalExtension) + : AbstractIOHandler{path, at} + , m_impl{this, std::move(jsonCfg), format, std::move(originalExtension)} {} std::future JSONIOHandler::flush(internal::ParsedFlushParams &) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index bec6cebb71..73d50366f7 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -26,8 +26,12 @@ #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/Writable.hpp" +#include + +#include #include #include #include @@ -54,9 +58,82 @@ namespace openPMD throw std::runtime_error((TEXT)); \ } -JSONIOHandlerImpl::JSONIOHandlerImpl(AbstractIOHandler *handler) +namespace +{ + struct DefaultValue + { + template + static nlohmann::json call() + { + if constexpr (auxiliary::IsComplex_v) + { + return typename T::value_type{}; + } + else + { + return T{}; + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + static constexpr char const *errorMsg = "JSON default value"; + }; + + /* + * If initializeWithDefaultValue contains a datatype, then the dataset ought + * to be initialized with the zero value of that dataset. + * Otherwise with null. + */ + nlohmann::json initializeNDArray( + Extent const &extent, + std::optional initializeWithDefaultValue) + { + // idea: begin from the innermost shale and copy the result into the + // outer shales + nlohmann::json accum = initializeWithDefaultValue.has_value() + ? switchNonVectorType( + initializeWithDefaultValue.value()) + : nlohmann::json(); + nlohmann::json old; + auto *accum_ptr = &accum; + auto *old_ptr = &old; + for (auto it = extent.rbegin(); it != extent.rend(); it++) + { + std::swap(old_ptr, accum_ptr); + *accum_ptr = nlohmann::json::array(); + for (Extent::value_type i = 0; i < *it; i++) + { + (*accum_ptr)[i] = *old_ptr; // copy boi + } + } + return *accum_ptr; + } +} // namespace + +JSONIOHandlerImpl::JSONIOHandlerImpl( + AbstractIOHandler *handler, + openPMD::json::TracingJSON config, + FileFormat format, + std::string originalExtension) : AbstractIOHandlerImpl(handler) -{} + , m_fileFormat{format} + , m_originalExtension{std::move(originalExtension)} +{ + // Currently unused + (void)config; +} JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; @@ -80,11 +157,7 @@ void JSONIOHandlerImpl::createFile( if (!writable->written) { - std::string name = parameters.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameters.name + m_originalExtension; auto res_pair = getPossiblyExisting(name); auto fullPathToFile = fullPath(std::get<0>(res_pair)); @@ -205,20 +278,23 @@ void JSONIOHandlerImpl::createDataset( setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; dset["datatype"] = datatypeToString(parameter.dtype); + auto extent = parameter.extent; switch (parameter.dtype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - auto complexExtent = parameter.extent; - complexExtent.push_back(2); - dset["data"] = initializeNDArray(complexExtent); + extent.push_back(2); break; } default: - dset["data"] = initializeNDArray(parameter.extent); break; } + // TOML does not support nulls, so initialize with zero + dset["data"] = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : parameter.dtype); writable->written = true; m_dirty.emplace(file); } @@ -276,27 +352,28 @@ void JSONIOHandlerImpl::extendDataset( throw std::runtime_error( "[JSON] The specified location contains no valid dataset"); } - switch (stringToDatatype(j["datatype"].get())) + auto extent = parameters.extent; + auto datatype = stringToDatatype(j["datatype"].get()); + switch (datatype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - // @todo test complex resizing - auto complexExtent = parameters.extent; - complexExtent.push_back(2); - nlohmann::json newData = initializeNDArray(complexExtent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + extent.push_back(2); break; } default: - nlohmann::json newData = initializeNDArray(parameters.extent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + // nothing to do break; } + // TOML does not support nulls, so initialize with zero + nlohmann::json newData = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : datatype); + nlohmann::json &oldData = j["data"]; + mergeInto(newData, oldData); + j["data"] = newData; writable->written = true; } @@ -521,11 +598,7 @@ void JSONIOHandlerImpl::openFile( "Supplied directory is not valid: " + m_handler->directory); } - std::string name = parameter.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameter.name + m_originalExtension; auto file = std::get<0>(getPossiblyExisting(name)); @@ -844,7 +917,8 @@ void JSONIOHandlerImpl::readAttribute( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto name = removeSlashes(parameters.name); - auto &jsonLoc = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + auto const &jsonLoc = jsonContents["attributes"]; setAndGetFilePosition(writable); std::string error_msg("[JSON] No such attribute '"); if (!hasKey(jsonLoc, name)) @@ -919,7 +993,12 @@ void JSONIOHandlerImpl::listAttributes( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto filePosition = setAndGetFilePosition(writable); - auto &j = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + if (!jsonContents.contains("attributes")) + { + return; + } + auto const &j = jsonContents["attributes"]; for (auto it = j.begin(); it != j.end(); it++) { parameters.attributes->push_back(it.key()); @@ -932,14 +1011,16 @@ void JSONIOHandlerImpl::deregister( m_files.erase(writable); } -std::shared_ptr -JSONIOHandlerImpl::getFilehandle(File fileName, Access access) +auto JSONIOHandlerImpl::getFilehandle(File fileName, Access access) + -> std::tuple, std::istream *, std::ostream *> { VERIFY_ALWAYS( fileName.valid(), "[JSON] Tried opening a file that has been overwritten or deleted.") auto path = fullPath(std::move(fileName)); - auto fs = std::make_shared(); + auto fs = std::make_unique(); + std::istream *istream = nullptr; + std::ostream *ostream = nullptr; if (access::write(access)) { /* @@ -949,14 +1030,31 @@ JSONIOHandlerImpl::getFilehandle(File fileName, Access access) * equivalent, but the openPMD frontend exposes no reading * functionality in APPEND mode. */ - fs->open(path, std::ios_base::out | std::ios_base::trunc); + std::ios_base::openmode openmode = + std::ios_base::out | std::ios_base::trunc; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + ostream = + &(*fs << std::setprecision( + std::numeric_limits::digits10 + 1)); } else { - fs->open(path, std::ios_base::in); + std::ios_base::openmode openmode = std::ios_base::in; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + istream = + &(*fs >> + std::setprecision(std::numeric_limits::digits10 + 1)); } VERIFY(fs->good(), "[JSON] Failed opening a file '" + path + "'"); - return fs; + return std::make_tuple(std::move(fs), istream, ostream); } std::string JSONIOHandlerImpl::fullPath(File fileName) @@ -1048,26 +1146,6 @@ Extent JSONIOHandlerImpl::getMultiplicators(Extent const &extent) return res; } -nlohmann::json JSONIOHandlerImpl::initializeNDArray(Extent const &extent) -{ - // idea: begin from the innermost shale and copy the result into the - // outer shales - nlohmann::json accum; - nlohmann::json old; - auto *accum_ptr = &accum; - auto *old_ptr = &old; - for (auto it = extent.rbegin(); it != extent.rend(); it++) - { - std::swap(old_ptr, accum_ptr); - *accum_ptr = nlohmann::json{}; - for (Extent::value_type i = 0; i < *it; i++) - { - (*accum_ptr)[i] = *old_ptr; // copy boi - } - } - return *accum_ptr; -} - Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j) { Extent res; @@ -1106,7 +1184,7 @@ std::string JSONIOHandlerImpl::removeSlashes(std::string s) } template -bool JSONIOHandlerImpl::hasKey(nlohmann::json &j, KeyT &&key) +bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) { return j.find(std::forward(key)) != j.end(); } @@ -1166,9 +1244,19 @@ std::shared_ptr JSONIOHandlerImpl::obtainJsonContents(File file) return it->second; } // read from file - auto fh = getFilehandle(file, Access::READ_ONLY); + auto [fh, fh_with_precision, _] = getFilehandle(file, Access::READ_ONLY); + (void)_; std::shared_ptr res = std::make_shared(); - *fh >> *res; + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision >> *res; + break; + case FileFormat::Toml: + *res = + openPMD::json::tomlToJson(toml::parse(*fh_with_precision, *file)); + break; + } VERIFY(fh->good(), "[JSON] Failed reading from a file."); m_jsonVals.emplace(file, res); return res; @@ -1192,9 +1280,22 @@ void JSONIOHandlerImpl::putJsonContents( auto it = m_jsonVals.find(filename); if (it != m_jsonVals.end()) { - auto fh = getFilehandle(filename, Access::CREATE); + auto [fh, _, fh_with_precision] = + getFilehandle(filename, Access::CREATE); + (void)_; (*it->second)["platform_byte_widths"] = platformSpecifics(); - *fh << *it->second << std::endl; + + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision << *it->second << std::endl; + break; + case FileFormat::Toml: + *fh_with_precision << openPMD::json::jsonToToml(*it->second) + << std::endl; + break; + } + VERIFY(fh->good(), "[JSON] Failed writing data to disk.") m_jsonVals.erase(it); if (unsetDirty) @@ -1399,7 +1500,7 @@ void JSONIOHandlerImpl::AttributeWriter::call( template void JSONIOHandlerImpl::AttributeReader::call( - nlohmann::json &json, Parameter ¶meters) + nlohmann::json const &json, Parameter ¶meters) { JsonToCpp jtc; *parameters.resource = jtc(json); diff --git a/src/Series.cpp b/src/Series.cpp index 4a22330179..59a96efdb7 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2180,7 +2180,8 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) std::map const backendDescriptors{ {"hdf5", Format::HDF5}, {"adios2", Format::ADIOS2_BP}, - {"json", Format::JSON}}; + {"json", Format::JSON}, + {"toml", Format::TOML}}; std::string backend; getJsonOptionLowerCase(options, "backend", backend); if (!backend.empty()) diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index c04e672ae6..168cab7bf6 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -200,6 +200,7 @@ namespace return nlohmann::json(); // null } + // @todo maybe generalize error type throw error::BackendConfigSchema( currentPath, "Unexpected datatype in TOML configuration. This is probably a " @@ -215,7 +216,8 @@ namespace switch (val.type()) { case nlohmann::json::value_t::null: - return toml::value(); + throw error::BackendConfigSchema( + currentPath, "TOML does not support null values."); case nlohmann::json::value_t::object: { toml::value::table_type res; for (auto pair = val.begin(); pair != val.end(); ++pair) @@ -247,7 +249,7 @@ namespace case nlohmann::json::value_t::number_unsigned: return val.get(); case nlohmann::json::value_t::number_float: - return val.get(); + return (long double)val.get(); case nlohmann::json::value_t::binary: return val.get(); case nlohmann::json::value_t::discarded: @@ -501,7 +503,7 @@ std::optional asLowerCaseStringDynamic(nlohmann::json const &value) std::vector backendKeys() { - return {"adios2", "json", "hdf5"}; + return {"adios2", "json", "toml", "hdf5"}; } void warnGlobalUnusedOptions(TracingJSON const &config) diff --git a/src/config.cpp b/src/config.cpp index a44925287a..89a824500c 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -28,20 +28,31 @@ #include #include +// @todo add TOML here std::map openPMD::getVariants() { + // clang-format off return std::map{ {"mpi", bool(openPMD_HAVE_MPI)}, {"json", true}, +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 + {"toml", true}, +#endif {"hdf5", bool(openPMD_HAVE_HDF5)}, {"adios1", false}, {"adios2", bool(openPMD_HAVE_ADIOS2)}}; + // clang-format on } std::vector openPMD::getFileExtensions() { std::vector fext; fext.emplace_back("json"); +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 + fext.emplace_back("toml"); +#endif #if openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 224596864e..d660e29ec4 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1051,7 +1051,7 @@ TEST_CASE("no_file_ending", "[core]") Access::CREATE, R"({"backend": "json"})"); } - REQUIRE(auxiliary::file_exists("../samples/no_extension_specified.json")); + REQUIRE(auxiliary::file_exists("../samples/no_extension_specified")); } TEST_CASE("backend_via_json", "[core]") diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 11297c7595..962ca636aa 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -81,20 +81,20 @@ std::vector testedFileExtensions() allExtensions.begin(), allExtensions.end(), []([[maybe_unused]] std::string const &ext) { -#if openPMD_HAVE_ADIOS2 -#define HAS_ADIOS_2_9 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) -#if HAS_ADIOS_2_9 +#if openPMD_HAS_ADIOS_2_9 // sst and ssc need a receiver for testing // bp5 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp5"; + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests + return ext == "sst" || ext == "ssc" || ext == "bp5" || + ext == "toml"; #else + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests // sst and ssc need a receiver for testing // bp4 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp4"; -#endif -#undef HAS_ADIOS_2_9 -#else - return false; + return ext == "sst" || ext == "ssc" || ext == "bp4" || + ext == "toml"; #endif }); return {allExtensions.begin(), newEnd}; @@ -1231,7 +1231,7 @@ TEST_CASE("particle_patches", "[serial]") inline void dtype_test(const std::string &backend) { - bool test_long_double = (backend != "json") || sizeof(long double) <= 8; + bool test_long_double = backend != "json" && backend != "toml"; bool test_long_long = (backend != "json") || sizeof(long long) <= 8; { Series s = Series("../samples/dtype_test." + backend, Access::CREATE); @@ -1447,7 +1447,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); REQUIRE(s.getAttribute("int").dtype == Datatype::INT); REQUIRE(s.getAttribute("long").dtype == Datatype::LONG); - REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + } REQUIRE(s.getAttribute("ushort").dtype == Datatype::USHORT); REQUIRE(s.getAttribute("uint").dtype == Datatype::UINT); REQUIRE(s.getAttribute("ulong").dtype == Datatype::ULONG); @@ -1459,7 +1462,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("vecShort").dtype == Datatype::VEC_SHORT); REQUIRE(s.getAttribute("vecInt").dtype == Datatype::VEC_INT); REQUIRE(s.getAttribute("vecLong").dtype == Datatype::VEC_LONG); - REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + } REQUIRE(s.getAttribute("vecUShort").dtype == Datatype::VEC_USHORT); REQUIRE(s.getAttribute("vecUInt").dtype == Datatype::VEC_UINT); REQUIRE(s.getAttribute("vecULong").dtype == Datatype::VEC_ULONG); @@ -1508,6 +1514,15 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + dtype_test("toml"); + } } inline void write_test(const std::string &backend) @@ -2106,6 +2121,15 @@ TEST_CASE("fileBased_write_test", "[serial]") { fileBased_write_test(t); } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + fileBased_write_test("toml"); + } } inline void sample_write_thetaMode(std::string file_ending) @@ -7405,4 +7429,13 @@ TEST_CASE("groupbased_read_write", "[serial]") } } } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + groupbased_read_write("toml"); + } } diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 0aa71cf7a8..a510098c8d 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -359,7 +359,8 @@ def attributeRoundTrip(self, file_ending): # c_types self.assertEqual(series.get_attribute("byte_c"), 30) self.assertEqual(series.get_attribute("ubyte_c"), 50) - if file_ending != "json": # TODO: returns [100] instead of 100 in json + # TODO: returns [100] instead of 100 in json/toml + if file_ending != "json" and file_ending != "toml": self.assertEqual(chr(series.get_attribute("char_c")), 'd') self.assertEqual(series.get_attribute("int16_c"), 2) self.assertEqual(series.get_attribute("int32_c"), 3) @@ -1824,7 +1825,7 @@ def testIterator(self): self.makeIteratorRoundTrip(b, backend_filesupport[b]) def makeAvailableChunksRoundTrip(self, ext): - if ext == "h5": + if ext == "h5" or ext == "toml": return name = "../samples/available_chunks_python." + ext write = io.Series( From 6bb7d82054c8adc91972c3349a20cfe9eda40383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 18 Aug 2023 11:53:26 +0200 Subject: [PATCH 06/13] Streaming examples: Set WAN as default transport (#1511) If ADIOS2 v2.9 is built against MPICH, it will pick MPI as its default streaming transport. This might break the CI on containers where MPICH does not actually support that. --- examples/10_streaming_read.cpp | 11 ++++++++++- examples/10_streaming_read.py | 3 ++- examples/10_streaming_write.cpp | 11 ++++++++++- examples/10_streaming_write.py | 3 ++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index 2128297934..6e6aba1fd9 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -19,7 +19,16 @@ int main() return 0; } - Series series = Series("electrons.sst", Access::READ_LINEAR); + Series series = Series("electrons.sst", Access::READ_LINEAR, R"( +{ + "adios2": { + "engine": { + "parameters": { + "DataTransport": "WAN" + } + } + } +})"); // `Series::writeIterations()` and `Series::readIterations()` are // intentionally restricted APIs that ensure a workflow which also works diff --git a/examples/10_streaming_read.py b/examples/10_streaming_read.py index 4946c36e04..f33d778842 100755 --- a/examples/10_streaming_read.py +++ b/examples/10_streaming_read.py @@ -7,7 +7,8 @@ # pass-through for ADIOS2 engine parameters # https://adios2.readthedocs.io/en/latest/engines/engines.html config = {'adios2': {'engine': {}, 'dataset': {}}} -config['adios2']['engine'] = {'parameters': {'Threads': '4'}} +config['adios2']['engine'] = {'parameters': + {'Threads': '4', 'DataTransport': 'WAN'}} config['adios2']['dataset'] = {'operators': [{'type': 'bzip2'}]} if __name__ == "__main__": diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 463b4c52e2..2eb825ae4a 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -20,7 +20,16 @@ int main() } // open file for writing - Series series = Series("electrons.sst", Access::CREATE); + Series series = Series("electrons.sst", Access::CREATE, R"( +{ + "adios2": { + "engine": { + "parameters": { + "DataTransport": "WAN" + } + } + } +})"); Datatype datatype = determineDatatype(); constexpr unsigned long length = 10ul; diff --git a/examples/10_streaming_write.py b/examples/10_streaming_write.py index bf92bcf14c..e0b830bd35 100755 --- a/examples/10_streaming_write.py +++ b/examples/10_streaming_write.py @@ -8,7 +8,8 @@ # pass-through for ADIOS2 engine parameters # https://adios2.readthedocs.io/en/latest/engines/engines.html config = {'adios2': {'engine': {}, 'dataset': {}}} -config['adios2']['engine'] = {'parameters': {'Threads': '4'}} +config['adios2']['engine'] = {'parameters': + {'Threads': '4', 'DataTransport': 'WAN'}} config['adios2']['dataset'] = {'operators': [{'type': 'bzip2'}]} if __name__ == "__main__": From 609fcdd93ae78af2a3762b8f8053193990bcc27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 18 Aug 2023 13:54:23 +0200 Subject: [PATCH 07/13] Replace openPMD_Datatypes global with function (#1509) --- include/openPMD/Datatype.hpp | 2 +- src/Datatype.cpp | 83 +++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 24e60c1d22..05d0ddefbb 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -92,7 +92,7 @@ enum class Datatype : int * listed in order in a vector. * */ -extern std::vector openPMD_Datatypes; +std::vector openPMD_Datatypes(); /** @brief Fundamental equivalence check for two given types T and U. * diff --git a/src/Datatype.cpp b/src/Datatype.cpp index 71565b3d52..f0f26f7ae9 100644 --- a/src/Datatype.cpp +++ b/src/Datatype.cpp @@ -224,46 +224,49 @@ std::string datatypeToString(openPMD::Datatype dt) return buf.str(); } -std::vector openPMD_Datatypes{ - Datatype::CHAR, - Datatype::UCHAR, - Datatype::SCHAR, - Datatype::SHORT, - Datatype::INT, - Datatype::LONG, - Datatype::LONGLONG, - Datatype::USHORT, - Datatype::UINT, - Datatype::ULONG, - Datatype::ULONGLONG, - Datatype::FLOAT, - Datatype::DOUBLE, - Datatype::LONG_DOUBLE, - Datatype::CFLOAT, - Datatype::CDOUBLE, - Datatype::CLONG_DOUBLE, - Datatype::STRING, - Datatype::VEC_CHAR, - Datatype::VEC_SHORT, - Datatype::VEC_INT, - Datatype::VEC_LONG, - Datatype::VEC_LONGLONG, - Datatype::VEC_UCHAR, - Datatype::VEC_USHORT, - Datatype::VEC_UINT, - Datatype::VEC_ULONG, - Datatype::VEC_ULONGLONG, - Datatype::VEC_FLOAT, - Datatype::VEC_DOUBLE, - Datatype::VEC_LONG_DOUBLE, - Datatype::VEC_CFLOAT, - Datatype::VEC_CDOUBLE, - Datatype::VEC_CLONG_DOUBLE, - Datatype::VEC_SCHAR, - Datatype::VEC_STRING, - Datatype::ARR_DBL_7, - Datatype::BOOL, - Datatype::UNDEFINED}; +std::vector openPMD_Datatypes() +{ + return { + Datatype::CHAR, + Datatype::UCHAR, + Datatype::SCHAR, + Datatype::SHORT, + Datatype::INT, + Datatype::LONG, + Datatype::LONGLONG, + Datatype::USHORT, + Datatype::UINT, + Datatype::ULONG, + Datatype::ULONGLONG, + Datatype::FLOAT, + Datatype::DOUBLE, + Datatype::LONG_DOUBLE, + Datatype::CFLOAT, + Datatype::CDOUBLE, + Datatype::CLONG_DOUBLE, + Datatype::STRING, + Datatype::VEC_CHAR, + Datatype::VEC_SHORT, + Datatype::VEC_INT, + Datatype::VEC_LONG, + Datatype::VEC_LONGLONG, + Datatype::VEC_UCHAR, + Datatype::VEC_USHORT, + Datatype::VEC_UINT, + Datatype::VEC_ULONG, + Datatype::VEC_ULONGLONG, + Datatype::VEC_FLOAT, + Datatype::VEC_DOUBLE, + Datatype::VEC_LONG_DOUBLE, + Datatype::VEC_CFLOAT, + Datatype::VEC_CDOUBLE, + Datatype::VEC_CLONG_DOUBLE, + Datatype::VEC_SCHAR, + Datatype::VEC_STRING, + Datatype::ARR_DBL_7, + Datatype::BOOL, + Datatype::UNDEFINED}; +} namespace { From e594d471625f15ad3ae9ff52e5ef6524475a3231 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 18 Aug 2023 12:13:32 -0400 Subject: [PATCH 08/13] CMake: Warn and Continue on Empty HDF5_VERSION (#1512) Seen on Conda-Forge for arm64 on macOS for HDF5 1.14.1 --- CMakeLists.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d1c0af5d91..c47f230507 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -331,10 +331,16 @@ endif() # HDF5 checks string(CONCAT openPMD_HDF5_STATUS "") # version: lower limit -if(openPMD_HAVE_HDF5 AND HDF5_VERSION VERSION_LESS 1.8.13) - string(CONCAT openPMD_HDF5_STATUS - "Found HDF5 version ${HDF5_VERSION} is too old. At least " - "version 1.8.13 is required.\n") +if(openPMD_HAVE_HDF5) + if(HDF5_VERSION STREQUAL "") + message(WARNING "HDF5_VERSION is empty. Now assuming it is 1.8.13 or newer.") + else() + if(HDF5_VERSION VERSION_LESS 1.8.13) + string(CONCAT openPMD_HDF5_STATUS + "Found HDF5 version ${HDF5_VERSION} is too old. At least " + "version 1.8.13 is required.\n") + endif() + endif() endif() # we imply support for parallel I/O if MPI variant is ON if(openPMD_HAVE_MPI AND openPMD_HAVE_HDF5 From 9d5bf968f5cd8bf01abfd20986c41160fee1f28a Mon Sep 17 00:00:00 2001 From: Ilian Kara-Mostefa <95044023+IlianCS@users.noreply.github.com> Date: Fri, 18 Aug 2023 09:13:43 -0700 Subject: [PATCH 09/13] replace extent in weighting and displacement (#1510) store extent value in n_particles --- examples/9_particle_write_serial.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/9_particle_write_serial.py b/examples/9_particle_write_serial.py index 0109a48a57..5af8796d49 100644 --- a/examples/9_particle_write_serial.py +++ b/examples/9_particle_write_serial.py @@ -16,7 +16,7 @@ if __name__ == "__main__": # open file for writing f = Series( - "../samples/7_particle_write_serial_py.h5", + "../samples/9_particle_write_serial_py.h5", Access.create ) @@ -35,27 +35,29 @@ "Electrons... the necessary evil for ion acceleration! ", "Just kidding.") + n_particles = 234 + # let's set a weird user-defined record this time electrons["displacement"].unit_dimension = {Unit_Dimension.M: 1} electrons["displacement"][SCALAR].unit_SI = 1.e-6 - dset = Dataset(np.dtype("float64"), extent=[2]) + dset = Dataset(np.dtype("float64"), extent=[n_particles]) electrons["displacement"][SCALAR].reset_dataset(dset) electrons["displacement"][SCALAR].make_constant(42.43) # don't like it anymore? remove it with: # del electrons["displacement"] electrons["weighting"][SCALAR] \ - .reset_dataset(Dataset(np.dtype("float32"), extent=[1])) \ + .reset_dataset(Dataset(np.dtype("float32"), extent=[n_particles])) \ .make_constant(1.e-5) - particlePos_x = np.random.rand(234).astype(np.float32) - particlePos_y = np.random.rand(234).astype(np.float32) + particlePos_x = np.random.rand(n_particles).astype(np.float32) + particlePos_y = np.random.rand(n_particles).astype(np.float32) d = Dataset(particlePos_x.dtype, extent=particlePos_x.shape) electrons["position"]["x"].reset_dataset(d) electrons["position"]["y"].reset_dataset(d) - particleOff_x = np.arange(234, dtype=np.uint) - particleOff_y = np.arange(234, dtype=np.uint) + particleOff_x = np.arange(n_particles, dtype=np.uint) + particleOff_y = np.arange(n_particles, dtype=np.uint) d = Dataset(particleOff_x.dtype, particleOff_x.shape) electrons["positionOffset"]["x"].reset_dataset(d) electrons["positionOffset"]["y"].reset_dataset(d) From acf1c11d640688e96a81cb876a51158ac03ef062 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 19 Aug 2023 21:54:31 -0600 Subject: [PATCH 10/13] Python: Fix ODR Violation (#1521) In pybind11 (and nanobind), auxiliary headers and `PYBIND11_MAKE_OPAQUE` definitions for distributed modules need to be included in every translation unit. Otherwise, a one-definition-rule violation occurs, which is undefined behavior. --- include/openPMD/binding/python/Common.hpp | 55 +++++++++++++++++++++ include/openPMD/binding/python/Numpy.hpp | 4 +- include/openPMD/binding/python/Pickle.hpp | 5 +- src/binding/python/Access.cpp | 6 +-- src/binding/python/Attributable.cpp | 10 +--- src/binding/python/BaseRecord.cpp | 8 +-- src/binding/python/BaseRecordComponent.cpp | 11 ++--- src/binding/python/ChunkInfo.cpp | 8 +-- src/binding/python/Container.cpp | 42 +--------------- src/binding/python/Dataset.cpp | 8 +-- src/binding/python/Datatype.cpp | 8 +-- src/binding/python/Error.cpp | 13 +++-- src/binding/python/Helper.cpp | 8 +-- src/binding/python/Iteration.cpp | 8 +-- src/binding/python/IterationEncoding.cpp | 6 +-- src/binding/python/Mesh.cpp | 8 +-- src/binding/python/MeshRecordComponent.cpp | 9 ++-- src/binding/python/ParticlePatches.cpp | 8 +-- src/binding/python/ParticleSpecies.cpp | 8 +-- src/binding/python/PatchRecord.cpp | 9 ++-- src/binding/python/PatchRecordComponent.cpp | 9 ++-- src/binding/python/Record.cpp | 8 +-- src/binding/python/RecordComponent.cpp | 11 ++--- src/binding/python/Series.cpp | 12 ++--- src/binding/python/UnitDimension.cpp | 6 +-- src/binding/python/openPMD.cpp | 7 +-- 26 files changed, 113 insertions(+), 182 deletions(-) create mode 100644 include/openPMD/binding/python/Common.hpp diff --git a/include/openPMD/binding/python/Common.hpp b/include/openPMD/binding/python/Common.hpp new file mode 100644 index 0000000000..39fed57238 --- /dev/null +++ b/include/openPMD/binding/python/Common.hpp @@ -0,0 +1,55 @@ +/* Copyright 2023 The openPMD Community + * + * This header is used to centrally define classes that shall not violate the + * C++ one-definition-rule (ODR) for various Python translation units. + * + * Authors: Axel Huebl + * License: LGPL-3.0-or-later + */ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" + +#include +#include +#include +#include +#include +// not yet used: +// pybind11/functional.h // for std::function + +// used exclusively in all our Python .cpp files +namespace py = pybind11; +using namespace openPMD; + +// opaque types +using PyIterationContainer = Series::IterationsContainer_t; +using PyMeshContainer = Container; +using PyPartContainer = Container; +using PyPatchContainer = Container; +using PyRecordContainer = Container; +using PyPatchRecordContainer = Container; +using PyRecordComponentContainer = Container; +using PyMeshRecordComponentContainer = Container; +using PyPatchRecordComponentContainer = Container; +using PyBaseRecordComponentContainer = Container; +PYBIND11_MAKE_OPAQUE(PyIterationContainer) +PYBIND11_MAKE_OPAQUE(PyMeshContainer) +PYBIND11_MAKE_OPAQUE(PyPartContainer) +PYBIND11_MAKE_OPAQUE(PyPatchContainer) +PYBIND11_MAKE_OPAQUE(PyRecordContainer) +PYBIND11_MAKE_OPAQUE(PyPatchRecordContainer) +PYBIND11_MAKE_OPAQUE(PyRecordComponentContainer) +PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) +PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) +PYBIND11_MAKE_OPAQUE(PyBaseRecordComponentContainer) diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index 3601ba7081..971aa8613f 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -22,9 +22,7 @@ #include "openPMD/Datatype.hpp" -#include -#include -#include +#include "Common.hpp" #include #include diff --git a/include/openPMD/binding/python/Pickle.hpp b/include/openPMD/binding/python/Pickle.hpp index a57a63d447..eabe307af9 100644 --- a/include/openPMD/binding/python/Pickle.hpp +++ b/include/openPMD/binding/python/Pickle.hpp @@ -24,8 +24,7 @@ #include "openPMD/Series.hpp" #include "openPMD/backend/Attributable.hpp" -#include -#include +#include "Common.hpp" #include #include @@ -46,8 +45,6 @@ template inline void add_pickle(pybind11::class_ &cl, T_SeriesAccessor &&seriesAccessor) { - namespace py = pybind11; - // helper: get first class in py::class_ - that's the type we pickle using PickledClass = typename std::tuple_element<0, std::tuple >::type; diff --git a/src/binding/python/Access.cpp b/src/binding/python/Access.cpp index 8fcdcb73c7..3b1d2a4297 100644 --- a/src/binding/python/Access.cpp +++ b/src/binding/python/Access.cpp @@ -18,13 +18,9 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/IO/Access.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_Access(py::module &m) { diff --git a/src/binding/python/Attributable.cpp b/src/binding/python/Attributable.cpp index a18ae2c471..88dbb95cbc 100644 --- a/src/binding/python/Attributable.cpp +++ b/src/binding/python/Attributable.cpp @@ -22,11 +22,9 @@ #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attribute.hpp" -#include "openPMD/binding/python/Numpy.hpp" -#include -#include -#include +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Numpy.hpp" #include #include @@ -35,11 +33,7 @@ #include #include -namespace py = pybind11; -using namespace openPMD; - using PyAttributeKeys = std::vector; -// PYBIND11_MAKE_OPAQUE(PyAttributeKeys) bool setAttributeFromBufferInfo( Attributable &attr, std::string const &key, py::buffer &a) diff --git a/src/binding/python/BaseRecord.cpp b/src/binding/python/BaseRecord.cpp index e893b54174..7aeee2d38a 100644 --- a/src/binding/python/BaseRecord.cpp +++ b/src/binding/python/BaseRecord.cpp @@ -18,18 +18,14 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/backend/BaseRecord.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" -#include "openPMD/binding/python/UnitDimension.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/UnitDimension.hpp" void init_BaseRecord(py::module &m) { diff --git a/src/binding/python/BaseRecordComponent.cpp b/src/binding/python/BaseRecordComponent.cpp index 0a3a306e76..95be596560 100644 --- a/src/binding/python/BaseRecordComponent.cpp +++ b/src/binding/python/BaseRecordComponent.cpp @@ -18,19 +18,14 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include -#include - -#include "openPMD/Datatype.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/Datatype.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" #include -namespace py = pybind11; -using namespace openPMD; - void init_BaseRecordComponent(py::module &m) { py::class_(m, "Base_Record_Component") diff --git a/src/binding/python/ChunkInfo.cpp b/src/binding/python/ChunkInfo.cpp index 5edc29353b..d86a579905 100644 --- a/src/binding/python/ChunkInfo.cpp +++ b/src/binding/python/ChunkInfo.cpp @@ -18,17 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/ChunkInfo.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Chunk(py::module &m) { py::class_(m, "ChunkInfo") diff --git a/src/binding/python/Container.cpp b/src/binding/python/Container.cpp index 8adebc570d..ef0a194637 100644 --- a/src/binding/python/Container.cpp +++ b/src/binding/python/Container.cpp @@ -23,23 +23,9 @@ * * BSD-style license, see pybind11 LICENSE file. */ - -#include -#include -#include - -#include "openPMD/Iteration.hpp" -#include "openPMD/Mesh.hpp" -#include "openPMD/ParticlePatches.hpp" -#include "openPMD/ParticleSpecies.hpp" -#include "openPMD/Record.hpp" -#include "openPMD/Series.hpp" -#include "openPMD/backend/BaseRecord.hpp" -#include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/backend/Container.hpp" -#include "openPMD/backend/MeshRecordComponent.hpp" -#include "openPMD/backend/PatchRecord.hpp" -#include "openPMD/backend/PatchRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include #include @@ -47,9 +33,6 @@ #include #include -namespace py = pybind11; -using namespace openPMD; - namespace detail { /* based on std_bind.h in pybind11 @@ -156,27 +139,6 @@ bind_container(py::handle scope, std::string const &name, Args &&...args) } } // namespace detail -using PyIterationContainer = Series::IterationsContainer_t; -using PyMeshContainer = Container; -using PyPartContainer = Container; -using PyPatchContainer = Container; -using PyRecordContainer = Container; -using PyPatchRecordContainer = Container; -using PyRecordComponentContainer = Container; -using PyMeshRecordComponentContainer = Container; -using PyPatchRecordComponentContainer = Container; -using PyBaseRecordComponentContainer = Container; -PYBIND11_MAKE_OPAQUE(PyIterationContainer) -PYBIND11_MAKE_OPAQUE(PyMeshContainer) -PYBIND11_MAKE_OPAQUE(PyPartContainer) -PYBIND11_MAKE_OPAQUE(PyPatchContainer) -PYBIND11_MAKE_OPAQUE(PyRecordContainer) -PYBIND11_MAKE_OPAQUE(PyPatchRecordContainer) -PYBIND11_MAKE_OPAQUE(PyRecordComponentContainer) -PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) -PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) -PYBIND11_MAKE_OPAQUE(PyBaseRecordComponentContainer) - void init_Container(py::module &m) { ::detail::bind_container(m, "Iteration_Container"); diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index bdd35db956..a67d7f1221 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -18,17 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Dataset.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" #include -namespace py = pybind11; -using namespace openPMD; - void init_Dataset(py::module &m) { py::class_(m, "Dataset") diff --git a/src/binding/python/Datatype.cpp b/src/binding/python/Datatype.cpp index 6c5587725d..9d53fba5ca 100644 --- a/src/binding/python/Datatype.cpp +++ b/src/binding/python/Datatype.cpp @@ -18,14 +18,10 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Datatype.hpp" -#include "openPMD/binding/python/Numpy.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Numpy.hpp" void init_Datatype(py::module &m) { diff --git a/src/binding/python/Error.cpp b/src/binding/python/Error.cpp index df1b39a5b2..681398c579 100644 --- a/src/binding/python/Error.cpp +++ b/src/binding/python/Error.cpp @@ -1,9 +1,14 @@ +/* Copyright 2019-2023 The openPMD Community + * + * This header is used to centrally define classes that shall not violate the + * C++ one-definition-rule (ODR) for various Python translation units. + * + * Authors: Franz Poeschel, Axel Huebl + * License: LGPL-3.0-or-later + */ #include "openPMD/Error.hpp" -#include - -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_Error(py::module &m) { diff --git a/src/binding/python/Helper.cpp b/src/binding/python/Helper.cpp index 9b07a37f70..e0aef84e0d 100644 --- a/src/binding/python/Helper.cpp +++ b/src/binding/python/Helper.cpp @@ -18,20 +18,16 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Series.hpp" #include "openPMD/cli/ls.hpp" #include "openPMD/helper/list_series.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Helper(py::module &m) { m.def( diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index b54c81c626..22e51aa974 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -18,18 +18,14 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Iteration.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Iteration(py::module &m) { py::class_(m, "Iteration") diff --git a/src/binding/python/IterationEncoding.cpp b/src/binding/python/IterationEncoding.cpp index 479ef65555..ef412e2639 100644 --- a/src/binding/python/IterationEncoding.cpp +++ b/src/binding/python/IterationEncoding.cpp @@ -18,13 +18,9 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/IterationEncoding.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_IterationEncoding(py::module &m) { diff --git a/src/binding/python/Mesh.cpp b/src/binding/python/Mesh.cpp index 8b6da30e64..e883e5e496 100644 --- a/src/binding/python/Mesh.cpp +++ b/src/binding/python/Mesh.cpp @@ -18,21 +18,17 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Mesh.hpp" #include "openPMD/backend/BaseRecord.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/binding/python/UnitDimension.hpp" #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Mesh(py::module &m) { py::class_ > cl(m, "Mesh"); diff --git a/src/binding/python/MeshRecordComponent.cpp b/src/binding/python/MeshRecordComponent.cpp index 98e602bbcb..120fe00da2 100644 --- a/src/binding/python/MeshRecordComponent.cpp +++ b/src/binding/python/MeshRecordComponent.cpp @@ -18,20 +18,17 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include +#include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" -#include "openPMD/backend/MeshRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include #include -namespace py = pybind11; -using namespace openPMD; - void init_MeshRecordComponent(py::module &m) { py::class_ cl( diff --git a/src/binding/python/ParticlePatches.cpp b/src/binding/python/ParticlePatches.cpp index 28deeab1b5..326162191d 100644 --- a/src/binding/python/ParticlePatches.cpp +++ b/src/binding/python/ParticlePatches.cpp @@ -18,17 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/ParticlePatches.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/PatchRecord.hpp" -#include +#include "openPMD/binding/python/Common.hpp" -namespace py = pybind11; -using namespace openPMD; +#include void init_ParticlePatches(py::module &m) { diff --git a/src/binding/python/ParticleSpecies.cpp b/src/binding/python/ParticleSpecies.cpp index 349081ea8e..349ea8c689 100644 --- a/src/binding/python/ParticleSpecies.cpp +++ b/src/binding/python/ParticleSpecies.cpp @@ -18,22 +18,18 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/ParticleSpecies.hpp" #include "openPMD/Record.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/Container.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include #include #include -namespace py = pybind11; -using namespace openPMD; - void init_ParticleSpecies(py::module &m) { py::class_ > cl(m, "ParticleSpecies"); diff --git a/src/binding/python/PatchRecord.cpp b/src/binding/python/PatchRecord.cpp index 7d01add8ea..aaef39546c 100644 --- a/src/binding/python/PatchRecord.cpp +++ b/src/binding/python/PatchRecord.cpp @@ -18,16 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include +#include "openPMD/backend/PatchRecord.hpp" #include "openPMD/backend/BaseRecord.hpp" -#include "openPMD/backend/PatchRecord.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" -#include "openPMD/binding/python/UnitDimension.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/UnitDimension.hpp" void init_PatchRecord(py::module &m) { diff --git a/src/binding/python/PatchRecordComponent.cpp b/src/binding/python/PatchRecordComponent.cpp index bf6a687b04..e668f7a1d9 100644 --- a/src/binding/python/PatchRecordComponent.cpp +++ b/src/binding/python/PatchRecordComponent.cpp @@ -18,16 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include +#include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" -#include "openPMD/backend/PatchRecordComponent.hpp" -#include "openPMD/binding/python/Numpy.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Numpy.hpp" namespace { diff --git a/src/binding/python/Record.cpp b/src/binding/python/Record.cpp index ee96e304e6..d97641bbce 100644 --- a/src/binding/python/Record.cpp +++ b/src/binding/python/Record.cpp @@ -18,21 +18,17 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Record.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/backend/BaseRecord.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/binding/python/UnitDimension.hpp" #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Record(py::module &m) { py::class_ > cl(m, "Record"); diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 32e1e0fb0d..ee8bb7d3bd 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -18,15 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include -#include - +#include "openPMD/RecordComponent.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" -#include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" #include "openPMD/binding/python/Pickle.hpp" @@ -42,9 +40,6 @@ #include #include -namespace py = pybind11; -using namespace openPMD; - /** Convert a py::tuple of py::slices to Offset & Extent * * https://docs.scipy.org/doc/numpy-1.15.0/reference/arrays.indexing.html diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 282b678c9e..241e14d69f 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -18,17 +18,14 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ - -#include -#include -#include - +#include "openPMD/Series.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/Series.hpp" #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/config.hpp" +#include "openPMD/binding/python/Common.hpp" + #if openPMD_HAVE_MPI // re-implemented signatures: // include @@ -38,9 +35,6 @@ #include #include -namespace py = pybind11; -using namespace openPMD; - #if openPMD_HAVE_MPI /** mpi4py communicator wrapper * diff --git a/src/binding/python/UnitDimension.cpp b/src/binding/python/UnitDimension.cpp index 72074cde78..6e46a6cfcf 100644 --- a/src/binding/python/UnitDimension.cpp +++ b/src/binding/python/UnitDimension.cpp @@ -18,13 +18,9 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/UnitDimension.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_UnitDimension(py::module &m) { diff --git a/src/binding/python/openPMD.cpp b/src/binding/python/openPMD.cpp index 5133c66d21..28b56b7d9c 100644 --- a/src/binding/python/openPMD.cpp +++ b/src/binding/python/openPMD.cpp @@ -18,18 +18,15 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include #include -namespace py = pybind11; - // forward declarations of exposed classes void init_Access(py::module &); void init_Attributable(py::module &); From 1cf46fd90cb4dbb7c1c50d9a371c3179717a212a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 19 Aug 2023 21:55:22 -0600 Subject: [PATCH 11/13] CI: Old CTest (#1519) CTest <3.20 does not support `--test-dir` and will then silently skip tests, because it cannot find any in the current directory. --- .github/workflows/linux.yml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index af47fd6380..a657d21825 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -37,7 +37,8 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure clang7_nopy_ompi_h5_ad2_libcpp: runs-on: ubuntu-20.04 @@ -74,11 +75,13 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + + cd build + ctest --output-on-failure find . -name *.bp | xargs -n1 -P1 -I {} rm -rf {} find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {} - ctest --test-dir build --output-on-failure + ctest --output-on-failure clang7_nopy_ompi_h5_ad2: runs-on: ubuntu-20.04 @@ -108,7 +111,8 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure # TODO # clang7_py36_nompi_h5_ad2_libstdc++ @@ -179,7 +183,8 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure # TODO: (old Travis-CI coverage) # clang10_py38_ompi_h5_1-10-6_ad2_release @@ -229,7 +234,8 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure gcc9_py38_pd_nompi_h5_ad2_libcpp: runs-on: ubuntu-20.04 @@ -251,7 +257,8 @@ jobs: -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure musllinux_py10: runs-on: ubuntu-20.04 @@ -276,7 +283,8 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DPython_EXECUTABLE=$(which python3.10) cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure conda_ompi_all: runs-on: ubuntu-20.04 @@ -310,4 +318,5 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure From 2384af0174a818c0abbe31bee148e45f464892f5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 19 Aug 2023 21:55:40 -0600 Subject: [PATCH 12/13] Fix `chmod` in `download_samples.sh` (#1518) The last entries are invalid. --- share/openPMD/download_samples.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/openPMD/download_samples.sh b/share/openPMD/download_samples.sh index 8691ce47a5..39599e9923 100755 --- a/share/openPMD/download_samples.sh +++ b/share/openPMD/download_samples.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # +set -eu -o pipefail # build directory as optional first argument, otherwise $PWD # we assume PWD is inside the CMake build directory @@ -46,9 +47,9 @@ rm -rf diags.zip diags # make sure we do not need write access when reading data chmod u-w samples/git-sample/*.h5 chmod u-w samples/git-sample/thetaMode/*.h5 -chmod u-w samples/samples/issue-sample/*.h5 -chmod u-w samples/samples/issue-sample/no_fields/*.h5 -chmod u-w samples/samples/issue-sample/no_particles/*.h5 +chmod u-w samples/issue-sample/*.h5 +chmod u-w samples/issue-sample/no_fields/*.h5 +chmod u-w samples/issue-sample/no_particles/*.h5 find samples/git-sample/3d-bp4 -type f -exec chmod u-w {} \; cd ${orgdir} From e01f72176c231b7e7e3433a0044df29e0ad5a3e2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 19 Aug 2023 21:56:16 -0600 Subject: [PATCH 13/13] Fix CMake: HDF5 Libs are PUBLIC (#1520) We do not yet use `FindHDF5.cmake` CMake targets and thus need to ensure the libs are propagated properly (they are not consumed necessarily as private) First seen when building with `-Wl,--disable-new-dtags` for RPATH over newer RUNPATH. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c47f230507..c3aaad3b1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -576,8 +576,9 @@ target_include_directories(openPMD SYSTEM PRIVATE $) # HDF5 Backend +# TODO: Once we require CMake 3.20+, simply link hdf5::hdf5 C lib target if(openPMD_HAVE_HDF5) - target_link_libraries(openPMD PRIVATE ${HDF5_LIBRARIES}) + target_link_libraries(openPMD PUBLIC ${HDF5_LIBRARIES}) target_include_directories(openPMD SYSTEM PRIVATE ${HDF5_INCLUDE_DIRS}) target_compile_definitions(openPMD PRIVATE ${HDF5_DEFINITIONS}) endif()