diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b9fa7b4f0b..3389b0c892 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -37,7 +37,8 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure clang7_nopy_ompi_h5_ad2_libcpp: runs-on: ubuntu-20.04 @@ -74,11 +75,13 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + + cd build + ctest --output-on-failure find . -name *.bp | xargs -n1 -P1 -I {} rm -rf {} find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {} - ctest --test-dir build --output-on-failure + ctest --output-on-failure clang7_nopy_ompi_h5_ad2: runs-on: ubuntu-20.04 @@ -108,7 +111,8 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure # TODO # clang7_py36_nompi_h5_ad2_libstdc++ @@ -179,7 +183,8 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure # TODO: (old Travis-CI coverage) # clang10_py38_ompi_h5_1-10-6_ad2_release @@ -229,7 +234,8 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure gcc9_py38_pd_nompi_h5_ad2_libcpp_julia: runs-on: ubuntu-20.04 @@ -265,7 +271,8 @@ jobs: cmake --build build --parallel 2 # Install the Julia side of CxxWrap julia --eval 'using Pkg; Pkg.add("CxxWrap")' - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure musllinux_py10: runs-on: ubuntu-20.04 @@ -290,7 +297,8 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DPython_EXECUTABLE=$(which python3.10) cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure conda_ompi_all: runs-on: ubuntu-20.04 @@ -324,4 +332,5 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + cd build + ctest --output-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d7665da04..4262c09075 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,7 +49,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.3 + rev: v1.5.4 hooks: - id: remove-tabs @@ -80,7 +80,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v2.2.1 + rev: v2.2.2 hooks: - id: pycln name: pycln (python) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a5e4258aa..4c8952ca3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,10 +336,16 @@ endif() # HDF5 checks string(CONCAT openPMD_HDF5_STATUS "") # version: lower limit -if(openPMD_HAVE_HDF5 AND HDF5_VERSION VERSION_LESS 1.8.13) - string(CONCAT openPMD_HDF5_STATUS - "Found HDF5 version ${HDF5_VERSION} is too old. At least " - "version 1.8.13 is required.\n") +if(openPMD_HAVE_HDF5) + if(HDF5_VERSION STREQUAL "") + message(WARNING "HDF5_VERSION is empty. Now assuming it is 1.8.13 or newer.") + else() + if(HDF5_VERSION VERSION_LESS 1.8.13) + string(CONCAT openPMD_HDF5_STATUS + "Found HDF5 version ${HDF5_VERSION} is too old. At least " + "version 1.8.13 is required.\n") + endif() + endif() endif() # we imply support for parallel I/O if MPI variant is ON if(openPMD_HAVE_MPI AND openPMD_HAVE_HDF5 @@ -415,7 +421,7 @@ if(CMAKE_VERSION VERSION_LESS 3.18.0) set(_PY_DEV_MODULE Development) endif() if(openPMD_USE_PYTHON STREQUAL AUTO) - find_package(Python 3.7.0 COMPONENTS Interpreter ${_PY_DEV_MODULE}) + find_package(Python 3.8.0 COMPONENTS Interpreter ${_PY_DEV_MODULE}) if(Python_FOUND) if(openPMD_USE_INTERNAL_PYBIND11) add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11") @@ -588,8 +594,9 @@ target_include_directories(openPMD SYSTEM PRIVATE $) # HDF5 Backend +# TODO: Once we require CMake 3.20+, simply link hdf5::hdf5 C lib target if(openPMD_HAVE_HDF5) - target_link_libraries(openPMD PRIVATE ${HDF5_LIBRARIES}) + target_link_libraries(openPMD PUBLIC ${HDF5_LIBRARIES}) target_include_directories(openPMD SYSTEM PRIVATE ${HDF5_INCLUDE_DIRS}) target_compile_definitions(openPMD PRIVATE ${HDF5_DEFINITIONS}) endif() diff --git a/Dockerfile b/Dockerfile index 54e9050987..5f08b571c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,8 @@ FROM quay.io/pypa/manylinux2010_x86_64 as build-env # FROM quay.io/pypa/manylinux1_x86_64 as build-env ENV DEBIAN_FRONTEND noninteractive -# Python 3.7-3.11 via "37m 38 39 311" -ARG PY_VERSIONS="37m 38 39 310 311" +# Python 3.8-3.11 via "38 39 311" +ARG PY_VERSIONS="38 39 310 311" # static libs need relocatable symbols for linking to shared python lib ENV CFLAGS="-fPIC ${CFLAGS}" @@ -112,30 +112,6 @@ RUN for whl in /opt/src/dist/*.whl; do \ && du -hs /opt/src/dist/* \ && du -hs /wheelhouse/* -# test in fresh env: Debian:Buster + Python 3.7 -FROM debian:buster -ENV DEBIAN_FRONTEND noninteractive -COPY --from=build-env /wheelhouse/openPMD_api-*-cp37-cp37m-manylinux2010_x86_64.whl . -RUN apt-get update \ - && apt-get install -y --no-install-recommends python3 python3-pip \ - && rm -rf /var/lib/apt/lists/* - # binutils -RUN python3 --version \ - && python3 -m pip install -U pip \ - && python3 -m pip install openPMD_api-*-cp37-cp37m-manylinux2010_x86_64.whl -RUN find / -name "openpmd*" -RUN ls -hal /usr/local/lib/python3.7/dist-packages/ -RUN ls -hal /usr/local/lib/python3.7/dist-packages/openpmd_api/ -# RUN ls -hal /usr/local/lib/python3.7/dist-packages/.libsopenpmd_api -# RUN objdump -x /usr/local/lib/python3.7/dist-packages/openpmd_api.cpython-37m-x86_64-linux-gnu.so | grep RPATH -RUN ldd /usr/local/lib/python3.7/dist-packages/openpmd_api/openpmd_api_cxx.cpython-37m-x86_64-linux-gnu.so -RUN python3 -c "import openpmd_api as io; print(io.__version__); print(io.variants)" -RUN python3 -m openpmd_api.ls --help -RUN openpmd-ls --help -#RUN echo "* soft core 100000" >> /etc/security/limits.conf && \ -# python3 -c "import openpmd_api as io"; \ -# gdb -ex bt -c core - # test in fresh env: Debian:Sid + Python 3.8 FROM debian:sid ENV DEBIAN_FRONTEND noninteractive diff --git a/README.md b/README.md index 85c224f055..fa2e5aca01 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ Optional language bindings: * Julia 1.7 - 1.10 * [libcxxwrap_julia](https://github.com/JuliaInterop/libcxxwrap-julia) 0.8.3 - 0.9.7 * Python: - * Python 3.7 - 3.11 + * Python 3.8 - 3.11 * pybind11 2.11.1+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/conda.yml b/conda.yml index 2bb1713efd..e82567b2ae 100644 --- a/conda.yml +++ b/conda.yml @@ -36,7 +36,7 @@ dependencies: - pre-commit - pyarrow # for dask # - pybind11 # shipped internally - - python>=3.7 + - python>=3.8 # just a note for later hackery, we could install pip packages inside the env, too: # - pip: diff --git a/docs/source/analysis/dask.rst b/docs/source/analysis/dask.rst index e00dce0e98..432bdd5790 100644 --- a/docs/source/analysis/dask.rst +++ b/docs/source/analysis/dask.rst @@ -41,6 +41,9 @@ The central Python API calls to convert to DASK datatypes are the ``ParticleSpec # note: no series.flush() needed +The ``to_dask_array`` method will automatically set Dask array chunking based on the available chunks in the read data set. +The default behavior can be overridden by passing an additional keyword argument ``chunks``, see the `dask.array.from_array documentation `__ for more details. +For example, to chunk only along the outermost axis in a 3D dataset using the default Dask array chunk size, call ``to_dask_array(chunks={0: 'auto', 1: -1, 2: -1})``. Example ------- diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 752497aa09..48ec6b1f44 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -1,10 +1,19 @@ .. _backends-json: -JSON -==== +JSON/TOML +========= -openPMD supports writing to and reading from JSON files. -The JSON backend is always available. +openPMD supports writing to and reading from JSON and TOML files. +The JSON and TOML backends are always available. + +.. note:: + + Both the JSON and the TOML backends are not intended for large-scale data I/O. + + The JSON backend is mainly intended for prototyping and learning, or similar workflows where setting up a large IO backend such as HDF5 or ADIOS2 is perceived as obstructive. It can also be used for small datasets that need to be stored in text format rather than binary. + + The TOML backend is intended for exchanging the *structure* of a data series without its "heavy" data fields. + For instance, one can easily create and exchange human-readable, machine-actionable data configurations for experiments and simulations. JSON File Format @@ -43,9 +52,17 @@ Every such attribute is itself a JSON object with two keys: * ``datatype``: A string describing the type of the value. * ``value``: The actual value of type ``datatype``. +TOML File Format +---------------- + +A TOML file uses the file ending ``.toml``. The TOML backend is chosen by creating a ``Series`` object with a filename that has this file ending. + +The TOML backend internally works with JSON datasets and converts to/from TOML during I/O. +As a result, data layout and usage are equivalent to the JSON backend. + -Restrictions ------------- +JSON Restrictions +----------------- For creation of JSON serializations (i.e. writing), the restrictions of the JSON backend are equivalent to those of the `JSON library by Niels Lohmann `_ @@ -77,6 +94,20 @@ The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved an A parallel (i.e. MPI) implementation is *not* available. +TOML Restrictions +----------------- + +Note that the JSON datatype-specific restrictions do not automatically hold for TOML, as those affect only the representation on disk, not the internal representation. + +TOML supports most numeric types, with the support for long double and long integer types being platform-defined. +Special floating point values such as NaN are also support. + +TOML does not support null values. + +The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved and must not be used for base/mesh/particles path, records and their components. + +A parallel (i.e. MPI) implementation is *not* available. + Example ------- diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index e82a330c8a..cc66b85587 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -39,7 +39,7 @@ Optional: language bindings * Python: - * Python 3.7 - 3.11 + * Python 3.8 - 3.11 * pybind11 2.11.1+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index 843bd2d76d..61ef593a2e 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -98,3 +98,9 @@ Attributes are (currently) unaffected by this: Some backends (e.g. the BP5 engine of ADIOS2) have multiple implementations for the openPMD-api-level guarantees of flush points. For user-guided selection of such implementations, ``Series::flush`` and ``Attributable::seriesFlush()`` take an optional JSON/TOML string as a parameter. See the section on :ref:`backend-specific configuration ` for details. + +Deferred Data API Contract +-------------------------- + +A verbose debug log can optionally be printed to the standard error output by specifying the environment variable ``OPENPMD_VERBOSE=1``. +Note that this functionality is at the current time still relatively basic. diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index 2128297934..6e6aba1fd9 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -19,7 +19,16 @@ int main() return 0; } - Series series = Series("electrons.sst", Access::READ_LINEAR); + Series series = Series("electrons.sst", Access::READ_LINEAR, R"( +{ + "adios2": { + "engine": { + "parameters": { + "DataTransport": "WAN" + } + } + } +})"); // `Series::writeIterations()` and `Series::readIterations()` are // intentionally restricted APIs that ensure a workflow which also works diff --git a/examples/10_streaming_read.py b/examples/10_streaming_read.py index 4946c36e04..f33d778842 100755 --- a/examples/10_streaming_read.py +++ b/examples/10_streaming_read.py @@ -7,7 +7,8 @@ # pass-through for ADIOS2 engine parameters # https://adios2.readthedocs.io/en/latest/engines/engines.html config = {'adios2': {'engine': {}, 'dataset': {}}} -config['adios2']['engine'] = {'parameters': {'Threads': '4'}} +config['adios2']['engine'] = {'parameters': + {'Threads': '4', 'DataTransport': 'WAN'}} config['adios2']['dataset'] = {'operators': [{'type': 'bzip2'}]} if __name__ == "__main__": diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 463b4c52e2..2eb825ae4a 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -20,7 +20,16 @@ int main() } // open file for writing - Series series = Series("electrons.sst", Access::CREATE); + Series series = Series("electrons.sst", Access::CREATE, R"( +{ + "adios2": { + "engine": { + "parameters": { + "DataTransport": "WAN" + } + } + } +})"); Datatype datatype = determineDatatype(); constexpr unsigned long length = 10ul; diff --git a/examples/10_streaming_write.py b/examples/10_streaming_write.py index bf92bcf14c..e0b830bd35 100755 --- a/examples/10_streaming_write.py +++ b/examples/10_streaming_write.py @@ -8,7 +8,8 @@ # pass-through for ADIOS2 engine parameters # https://adios2.readthedocs.io/en/latest/engines/engines.html config = {'adios2': {'engine': {}, 'dataset': {}}} -config['adios2']['engine'] = {'parameters': {'Threads': '4'}} +config['adios2']['engine'] = {'parameters': + {'Threads': '4', 'DataTransport': 'WAN'}} config['adios2']['dataset'] = {'operators': [{'type': 'bzip2'}]} if __name__ == "__main__": diff --git a/examples/9_particle_write_serial.py b/examples/9_particle_write_serial.py index 0109a48a57..5af8796d49 100644 --- a/examples/9_particle_write_serial.py +++ b/examples/9_particle_write_serial.py @@ -16,7 +16,7 @@ if __name__ == "__main__": # open file for writing f = Series( - "../samples/7_particle_write_serial_py.h5", + "../samples/9_particle_write_serial_py.h5", Access.create ) @@ -35,27 +35,29 @@ "Electrons... the necessary evil for ion acceleration! ", "Just kidding.") + n_particles = 234 + # let's set a weird user-defined record this time electrons["displacement"].unit_dimension = {Unit_Dimension.M: 1} electrons["displacement"][SCALAR].unit_SI = 1.e-6 - dset = Dataset(np.dtype("float64"), extent=[2]) + dset = Dataset(np.dtype("float64"), extent=[n_particles]) electrons["displacement"][SCALAR].reset_dataset(dset) electrons["displacement"][SCALAR].make_constant(42.43) # don't like it anymore? remove it with: # del electrons["displacement"] electrons["weighting"][SCALAR] \ - .reset_dataset(Dataset(np.dtype("float32"), extent=[1])) \ + .reset_dataset(Dataset(np.dtype("float32"), extent=[n_particles])) \ .make_constant(1.e-5) - particlePos_x = np.random.rand(234).astype(np.float32) - particlePos_y = np.random.rand(234).astype(np.float32) + particlePos_x = np.random.rand(n_particles).astype(np.float32) + particlePos_y = np.random.rand(n_particles).astype(np.float32) d = Dataset(particlePos_x.dtype, extent=particlePos_x.shape) electrons["position"]["x"].reset_dataset(d) electrons["position"]["y"].reset_dataset(d) - particleOff_x = np.arange(234, dtype=np.uint) - particleOff_y = np.arange(234, dtype=np.uint) + particleOff_x = np.arange(n_particles, dtype=np.uint) + particleOff_y = np.arange(n_particles, dtype=np.uint) d = Dataset(particleOff_x.dtype, particleOff_x.shape) electrons["positionOffset"]["x"].reset_dataset(d) electrons["positionOffset"]["y"].reset_dataset(d) diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 24e60c1d22..05d0ddefbb 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -92,7 +92,7 @@ enum class Datatype : int * listed in order in a vector. * */ -extern std::vector openPMD_Datatypes; +std::vector openPMD_Datatypes(); /** @brief Fundamental equivalence check for two given types T and U. * diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 1bca04f5d3..8d13f3feb8 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -26,7 +26,6 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include -#include namespace openPMD { @@ -36,198 +35,11 @@ class Writable; class AbstractIOHandlerImpl { public: - AbstractIOHandlerImpl(AbstractIOHandler *handler) : m_handler{handler} - {} + AbstractIOHandlerImpl(AbstractIOHandler *handler); virtual ~AbstractIOHandlerImpl() = default; - std::future flush() - { - using namespace auxiliary; - - while (!(*m_handler).m_work.empty()) - { - IOTask &i = (*m_handler).m_work.front(); - try - { - switch (i.operation) - { - using O = Operation; - case O::CREATE_FILE: - createFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CHECK_FILE: - checkFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CREATE_PATH: - createPath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CREATE_DATASET: - createDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::EXTEND_DATASET: - extendDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::OPEN_FILE: - openFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CLOSE_FILE: - closeFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::OPEN_PATH: - openPath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::CLOSE_PATH: - closePath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::OPEN_DATASET: - openDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_FILE: - deleteFile( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_PATH: - deletePath( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_DATASET: - deleteDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DELETE_ATT: - deleteAttribute( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::WRITE_DATASET: - writeDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::WRITE_ATT: - writeAttribute( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::READ_DATASET: - readDataset( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::GET_BUFFER_VIEW: - getBufferView( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::READ_ATT: - readAttribute( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::LIST_PATHS: - listPaths( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::LIST_DATASETS: - listDatasets( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::LIST_ATTS: - listAttributes( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::ADVANCE: - advance( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::AVAILABLE_CHUNKS: - availableChunks( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::KEEP_SYNCHRONOUS: - keepSynchronous( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - case O::DEREGISTER: - deregister( - i.writable, - deref_dynamic_cast >( - i.parameter.get())); - break; - } - } - catch (...) - { - std::cerr << "[AbstractIOHandlerImpl] IO Task " - << internal::operationAsString(i.operation) - << " failed with exception. Clearing IO queue and " - "passing on the exception." - << std::endl; - while (!m_handler->m_work.empty()) - { - m_handler->m_work.pop(); - } - throw; - } - (*m_handler).m_work.pop(); - } - return std::future(); - } + std::future flush(); /** * Close the file corresponding with the writable and release file handles. @@ -592,5 +404,10 @@ class AbstractIOHandlerImpl deregister(Writable *, Parameter const ¶m) = 0; AbstractIOHandler *m_handler; + bool m_verboseIOTasks = false; + + // Args will be forwarded to std::cerr if m_verboseIOTasks is true + template + void writeToStderr(Args &&...) const; }; // AbstractIOHandlerImpl } // namespace openPMD diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 43ec4d04a1..858da29a40 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -35,6 +35,7 @@ enum class Format ADIOS2_SST, ADIOS2_SSC, JSON, + TOML, DUMMY }; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 0cdc6f3c36..452098137e 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -29,7 +29,12 @@ namespace openPMD class JSONIOHandler : public AbstractIOHandler { public: - JSONIOHandler(std::string path, Access at); + JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON config, + JSONIOHandlerImpl::FileFormat, + std::string originalExtension); ~JSONIOHandler() override; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 7f10f62cd9..c935647665 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -26,8 +26,10 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/IO/JSON/JSONFilePosition.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/config.hpp" +#include #include #include @@ -153,7 +155,17 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl using json = nlohmann::json; public: - explicit JSONIOHandlerImpl(AbstractIOHandler *); + enum class FileFormat + { + Json, + Toml + }; + + explicit JSONIOHandlerImpl( + AbstractIOHandler *, + openPMD::json::TracingJSON config, + FileFormat, + std::string originalExtension); ~JSONIOHandlerImpl() override; @@ -229,15 +241,25 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // files that have logically, but not physically been written to std::unordered_set m_dirty; + /* + * Is set by constructor. + */ + FileFormat m_fileFormat{}; + + std::string m_originalExtension; + // HELPER FUNCTIONS - // will use the IOHandler to retrieve the correct directory - // shared pointer to circumvent the fact that c++ pre 17 does - // not enforce (only allow) copy elision in return statements - std::shared_ptr getFilehandle( - File, - Access access); //, Access - // m_frontendAccess=this->m_handler->m_frontendAccess); + // will use the IOHandler to retrieve the correct directory. + // first tuple element will be the underlying opened file handle. + // if Access is read mode, then the second tuple element will be the istream + // casted to precision std::numeric_limits::digits10 + 1, else null. + // if Access is write mode, then the second tuple element will be the + // ostream casted to precision std::numeric_limits::digits10 + 1, + // else null. first tuple element needs to be a pointer, since the casted + // streams are references only. + std::tuple, std::istream *, std::ostream *> + getFilehandle(File, Access access); // full operating system path of the given file std::string fullPath(File); @@ -272,15 +294,13 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // essentially: m_i = \prod_{j=0}^{i-1} extent_j static Extent getMultiplicators(Extent const &extent); - static nlohmann::json initializeNDArray(Extent const &extent); - static Extent getExtent(nlohmann::json &j); // remove single '/' in the beginning and end of a string static std::string removeSlashes(std::string); template - static bool hasKey(nlohmann::json &, KeyT &&key); + static bool hasKey(nlohmann::json const &, KeyT &&key); // make sure that the given path exists in proper form in // the passed json value @@ -366,7 +386,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl struct AttributeReader { template - static void call(nlohmann::json &, Parameter &); + static void + call(nlohmann::json const &, Parameter &); static constexpr char const *errorMsg = "JSON: writeAttribute"; }; diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 923cfb5be2..3e5a36774e 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -24,6 +24,7 @@ #include "openPMD/auxiliary/UniquePtr.hpp" #include +#include #include // size_t #include #include @@ -56,6 +57,18 @@ namespace detail static constexpr bool value = true; }; + template + struct IsComplex + { + static constexpr bool value = false; + }; + + template + struct IsComplex> + { + static constexpr bool value = true; + }; + template struct IsPointer { @@ -114,6 +127,9 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsContiguousContainer_v = IsVector_v || IsArray_v; +template +inline constexpr bool IsComplex_v = detail::IsComplex::value; + namespace { // see https://en.cppreference.com/w/cpp/language/if diff --git a/include/openPMD/binding/python/Common.hpp b/include/openPMD/binding/python/Common.hpp new file mode 100644 index 0000000000..39fed57238 --- /dev/null +++ b/include/openPMD/binding/python/Common.hpp @@ -0,0 +1,55 @@ +/* Copyright 2023 The openPMD Community + * + * This header is used to centrally define classes that shall not violate the + * C++ one-definition-rule (ODR) for various Python translation units. + * + * Authors: Axel Huebl + * License: LGPL-3.0-or-later + */ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" + +#include +#include +#include +#include +#include +// not yet used: +// pybind11/functional.h // for std::function + +// used exclusively in all our Python .cpp files +namespace py = pybind11; +using namespace openPMD; + +// opaque types +using PyIterationContainer = Series::IterationsContainer_t; +using PyMeshContainer = Container; +using PyPartContainer = Container; +using PyPatchContainer = Container; +using PyRecordContainer = Container; +using PyPatchRecordContainer = Container; +using PyRecordComponentContainer = Container; +using PyMeshRecordComponentContainer = Container; +using PyPatchRecordComponentContainer = Container; +using PyBaseRecordComponentContainer = Container; +PYBIND11_MAKE_OPAQUE(PyIterationContainer) +PYBIND11_MAKE_OPAQUE(PyMeshContainer) +PYBIND11_MAKE_OPAQUE(PyPartContainer) +PYBIND11_MAKE_OPAQUE(PyPatchContainer) +PYBIND11_MAKE_OPAQUE(PyRecordContainer) +PYBIND11_MAKE_OPAQUE(PyPatchRecordContainer) +PYBIND11_MAKE_OPAQUE(PyRecordComponentContainer) +PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) +PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) +PYBIND11_MAKE_OPAQUE(PyBaseRecordComponentContainer) diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index 3601ba7081..971aa8613f 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -22,9 +22,7 @@ #include "openPMD/Datatype.hpp" -#include -#include -#include +#include "Common.hpp" #include #include diff --git a/include/openPMD/binding/python/Pickle.hpp b/include/openPMD/binding/python/Pickle.hpp index a57a63d447..eabe307af9 100644 --- a/include/openPMD/binding/python/Pickle.hpp +++ b/include/openPMD/binding/python/Pickle.hpp @@ -24,8 +24,7 @@ #include "openPMD/Series.hpp" #include "openPMD/backend/Attributable.hpp" -#include -#include +#include "Common.hpp" #include #include @@ -46,8 +45,6 @@ template inline void add_pickle(pybind11::class_ &cl, T_SeriesAccessor &&seriesAccessor) { - namespace py = pybind11; - // helper: get first class in py::class_ - that's the type we pickle using PickledClass = typename std::tuple_element<0, std::tuple >::type; diff --git a/setup.py b/setup.py index 9db688042a..8d3a7e3348 100644 --- a/setup.py +++ b/setup.py @@ -192,7 +192,7 @@ def build_extension(self, ext): cmdclass=dict(build_ext=CMakeBuild), # scripts=['openpmd-ls'], zip_safe=False, - python_requires='>=3.7', + python_requires='>=3.8', # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli @@ -221,7 +221,6 @@ def build_extension(self, ext): 'Topic :: Database :: Front-Ends', 'Programming Language :: C++', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/share/openPMD/download_samples.sh b/share/openPMD/download_samples.sh index 8691ce47a5..39599e9923 100755 --- a/share/openPMD/download_samples.sh +++ b/share/openPMD/download_samples.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # +set -eu -o pipefail # build directory as optional first argument, otherwise $PWD # we assume PWD is inside the CMake build directory @@ -46,9 +47,9 @@ rm -rf diags.zip diags # make sure we do not need write access when reading data chmod u-w samples/git-sample/*.h5 chmod u-w samples/git-sample/thetaMode/*.h5 -chmod u-w samples/samples/issue-sample/*.h5 -chmod u-w samples/samples/issue-sample/no_fields/*.h5 -chmod u-w samples/samples/issue-sample/no_particles/*.h5 +chmod u-w samples/issue-sample/*.h5 +chmod u-w samples/issue-sample/no_fields/*.h5 +chmod u-w samples/issue-sample/no_particles/*.h5 find samples/git-sample/3d-bp4 -type f -exec chmod u-w {} \; cd ${orgdir} diff --git a/src/Datatype.cpp b/src/Datatype.cpp index 71565b3d52..f0f26f7ae9 100644 --- a/src/Datatype.cpp +++ b/src/Datatype.cpp @@ -224,46 +224,49 @@ std::string datatypeToString(openPMD::Datatype dt) return buf.str(); } -std::vector openPMD_Datatypes{ - Datatype::CHAR, - Datatype::UCHAR, - Datatype::SCHAR, - Datatype::SHORT, - Datatype::INT, - Datatype::LONG, - Datatype::LONGLONG, - Datatype::USHORT, - Datatype::UINT, - Datatype::ULONG, - Datatype::ULONGLONG, - Datatype::FLOAT, - Datatype::DOUBLE, - Datatype::LONG_DOUBLE, - Datatype::CFLOAT, - Datatype::CDOUBLE, - Datatype::CLONG_DOUBLE, - Datatype::STRING, - Datatype::VEC_CHAR, - Datatype::VEC_SHORT, - Datatype::VEC_INT, - Datatype::VEC_LONG, - Datatype::VEC_LONGLONG, - Datatype::VEC_UCHAR, - Datatype::VEC_USHORT, - Datatype::VEC_UINT, - Datatype::VEC_ULONG, - Datatype::VEC_ULONGLONG, - Datatype::VEC_FLOAT, - Datatype::VEC_DOUBLE, - Datatype::VEC_LONG_DOUBLE, - Datatype::VEC_CFLOAT, - Datatype::VEC_CDOUBLE, - Datatype::VEC_CLONG_DOUBLE, - Datatype::VEC_SCHAR, - Datatype::VEC_STRING, - Datatype::ARR_DBL_7, - Datatype::BOOL, - Datatype::UNDEFINED}; +std::vector openPMD_Datatypes() +{ + return { + Datatype::CHAR, + Datatype::UCHAR, + Datatype::SCHAR, + Datatype::SHORT, + Datatype::INT, + Datatype::LONG, + Datatype::LONGLONG, + Datatype::USHORT, + Datatype::UINT, + Datatype::ULONG, + Datatype::ULONGLONG, + Datatype::FLOAT, + Datatype::DOUBLE, + Datatype::LONG_DOUBLE, + Datatype::CFLOAT, + Datatype::CDOUBLE, + Datatype::CLONG_DOUBLE, + Datatype::STRING, + Datatype::VEC_CHAR, + Datatype::VEC_SHORT, + Datatype::VEC_INT, + Datatype::VEC_LONG, + Datatype::VEC_LONGLONG, + Datatype::VEC_UCHAR, + Datatype::VEC_USHORT, + Datatype::VEC_UINT, + Datatype::VEC_ULONG, + Datatype::VEC_ULONGLONG, + Datatype::VEC_FLOAT, + Datatype::VEC_DOUBLE, + Datatype::VEC_LONG_DOUBLE, + Datatype::VEC_CFLOAT, + Datatype::VEC_CDOUBLE, + Datatype::VEC_CLONG_DOUBLE, + Datatype::VEC_SCHAR, + Datatype::VEC_STRING, + Datatype::ARR_DBL_7, + Datatype::BOOL, + Datatype::UNDEFINED}; +} namespace { diff --git a/src/Format.cpp b/src/Format.cpp index d5a8acf5f3..8a6ead832a 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -43,6 +43,8 @@ Format determineFormat(std::string const &filename) return Format::ADIOS2_SSC; if (auxiliary::ends_with(filename, ".json")) return Format::JSON; + if (auxiliary::ends_with(filename, ".toml")) + return Format::TOML; // Format might still be specified via JSON return Format::DUMMY; @@ -66,6 +68,8 @@ std::string suffix(Format f) return ".ssc"; case Format::JSON: return ".json"; + case Format::TOML: + return ".toml"; default: return ""; } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 4cd74a4de2..c6cd69f8a5 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -194,7 +194,20 @@ std::unique_ptr createIOHandler( std::move(originalExtension)); case Format::JSON: return constructIOHandler( - "JSON", path, access); + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Json, + std::move(originalExtension)); + case Format::TOML: + return constructIOHandler( + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Toml, + std::move(originalExtension)); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending? Specified " diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index d762df6a1f..af827704a1 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -20,14 +20,365 @@ */ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" + +#include "openPMD/auxiliary/Environment.hpp" #include "openPMD/backend/Writable.hpp" +#include + namespace openPMD { +AbstractIOHandlerImpl::AbstractIOHandlerImpl(AbstractIOHandler *handler) + : m_handler{handler} +{ + if (auxiliary::getEnvNum("OPENPMD_VERBOSE", 0) != 0) + { + m_verboseIOTasks = true; + } +} + void AbstractIOHandlerImpl::keepSynchronous( Writable *writable, Parameter param) { writable->abstractFilePosition = param.otherWritable->abstractFilePosition; writable->written = true; } + +template +void AbstractIOHandlerImpl::writeToStderr([[maybe_unused]] Args &&...args) const +{ + if (m_verboseIOTasks) + { + (std::cerr << ... << args) << std::endl; + } +} + +std::future AbstractIOHandlerImpl::flush() +{ + using namespace auxiliary; + + while (!(*m_handler).m_work.empty()) + { + IOTask &i = (*m_handler).m_work.front(); + try + { + switch (i.operation) + { + using O = Operation; + case O::CREATE_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CREATE_FILE: ", + parameter.name); + createFile(i.writable, parameter); + break; + } + case O::CHECK_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CHECK_FILE: ", + parameter.name); + checkFile(i.writable, parameter); + break; + } + case O::CREATE_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CREATE_PATH: ", + parameter.path); + createPath(i.writable, parameter); + break; + } + case O::CREATE_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] CREATE_DATASET: ", + parameter.name); + createDataset(i.writable, parameter); + break; + } + case O::EXTEND_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] EXTEND_DATASET"); + extendDataset(i.writable, parameter); + break; + } + case O::OPEN_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] OPEN_FILE: ", + parameter.name); + openFile(i.writable, parameter); + break; + } + case O::CLOSE_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] CLOSE_FILE"); + closeFile(i.writable, parameter); + break; + } + case O::OPEN_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] OPEN_PATH: ", + parameter.path); + openPath(i.writable, parameter); + break; + } + case O::CLOSE_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] CLOSE_PATH"); + closePath(i.writable, parameter); + break; + } + case O::OPEN_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] OPEN_DATASET: ", + parameter.name); + openDataset(i.writable, parameter); + break; + } + case O::DELETE_FILE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DELETE_FILE"); + deleteFile(i.writable, parameter); + break; + } + case O::DELETE_PATH: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DELETE_PATH"); + deletePath(i.writable, parameter); + break; + } + case O::DELETE_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] DELETE_DATASET"); + deleteDataset(i.writable, parameter); + break; + } + case O::DELETE_ATT: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DELETE_ATT"); + deleteAttribute(i.writable, parameter); + break; + } + case O::WRITE_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] WRITE_DATASET"); + writeDataset(i.writable, parameter); + break; + } + case O::WRITE_ATT: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] WRITE_ATT: (", + parameter.dtype, + ") ", + parameter.name); + writeAttribute(i.writable, parameter); + break; + } + case O::READ_DATASET: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] READ_DATASET"); + readDataset(i.writable, parameter); + break; + } + case O::GET_BUFFER_VIEW: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] GET_BUFFER_VIEW"); + getBufferView(i.writable, parameter); + break; + } + case O::READ_ATT: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] READ_ATT: ", + parameter.name); + readAttribute(i.writable, parameter); + break; + } + case O::LIST_PATHS: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] LIST_PATHS"); + listPaths(i.writable, parameter); + break; + } + case O::LIST_DATASETS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] LIST_DATASETS"); + listDatasets(i.writable, parameter); + break; + } + case O::LIST_ATTS: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] LIST_ATTS"); + listAttributes(i.writable, parameter); + break; + } + case O::ADVANCE: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] ADVANCE"); + advance(i.writable, parameter); + break; + } + case O::AVAILABLE_CHUNKS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] AVAILABLE_CHUNKS"); + availableChunks(i.writable, parameter); + break; + } + case O::KEEP_SYNCHRONOUS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] KEEP_SYNCHRONOUS"); + keepSynchronous(i.writable, parameter); + break; + } + case O::DEREGISTER: { + auto ¶meter = deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", i.writable->parent, "->", i.writable, "] DEREGISTER"); + deregister(i.writable, parameter); + break; + } + } + } + catch (...) + { + std::cerr << "[AbstractIOHandlerImpl] IO Task " + << internal::operationAsString(i.operation) + << " failed with exception. Clearing IO queue and " + "passing on the exception." + << std::endl; + while (!m_handler->m_work.empty()) + { + m_handler->m_work.pop(); + } + throw; + } + (*m_handler).m_work.pop(); + } + return std::future(); +} } // namespace openPMD diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 10ffce9927..7eb8a57278 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -25,8 +25,14 @@ namespace openPMD { JSONIOHandler::~JSONIOHandler() = default; -JSONIOHandler::JSONIOHandler(std::string path, Access at) - : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} +JSONIOHandler::JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON jsonCfg, + JSONIOHandlerImpl::FileFormat format, + std::string originalExtension) + : AbstractIOHandler{path, at} + , m_impl{this, std::move(jsonCfg), format, std::move(originalExtension)} {} std::future JSONIOHandler::flush(internal::ParsedFlushParams &) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index bec6cebb71..73d50366f7 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -26,8 +26,12 @@ #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/Writable.hpp" +#include + +#include #include #include #include @@ -54,9 +58,82 @@ namespace openPMD throw std::runtime_error((TEXT)); \ } -JSONIOHandlerImpl::JSONIOHandlerImpl(AbstractIOHandler *handler) +namespace +{ + struct DefaultValue + { + template + static nlohmann::json call() + { + if constexpr (auxiliary::IsComplex_v) + { + return typename T::value_type{}; + } + else + { + return T{}; + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + static constexpr char const *errorMsg = "JSON default value"; + }; + + /* + * If initializeWithDefaultValue contains a datatype, then the dataset ought + * to be initialized with the zero value of that dataset. + * Otherwise with null. + */ + nlohmann::json initializeNDArray( + Extent const &extent, + std::optional initializeWithDefaultValue) + { + // idea: begin from the innermost shale and copy the result into the + // outer shales + nlohmann::json accum = initializeWithDefaultValue.has_value() + ? switchNonVectorType( + initializeWithDefaultValue.value()) + : nlohmann::json(); + nlohmann::json old; + auto *accum_ptr = &accum; + auto *old_ptr = &old; + for (auto it = extent.rbegin(); it != extent.rend(); it++) + { + std::swap(old_ptr, accum_ptr); + *accum_ptr = nlohmann::json::array(); + for (Extent::value_type i = 0; i < *it; i++) + { + (*accum_ptr)[i] = *old_ptr; // copy boi + } + } + return *accum_ptr; + } +} // namespace + +JSONIOHandlerImpl::JSONIOHandlerImpl( + AbstractIOHandler *handler, + openPMD::json::TracingJSON config, + FileFormat format, + std::string originalExtension) : AbstractIOHandlerImpl(handler) -{} + , m_fileFormat{format} + , m_originalExtension{std::move(originalExtension)} +{ + // Currently unused + (void)config; +} JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; @@ -80,11 +157,7 @@ void JSONIOHandlerImpl::createFile( if (!writable->written) { - std::string name = parameters.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameters.name + m_originalExtension; auto res_pair = getPossiblyExisting(name); auto fullPathToFile = fullPath(std::get<0>(res_pair)); @@ -205,20 +278,23 @@ void JSONIOHandlerImpl::createDataset( setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; dset["datatype"] = datatypeToString(parameter.dtype); + auto extent = parameter.extent; switch (parameter.dtype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - auto complexExtent = parameter.extent; - complexExtent.push_back(2); - dset["data"] = initializeNDArray(complexExtent); + extent.push_back(2); break; } default: - dset["data"] = initializeNDArray(parameter.extent); break; } + // TOML does not support nulls, so initialize with zero + dset["data"] = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : parameter.dtype); writable->written = true; m_dirty.emplace(file); } @@ -276,27 +352,28 @@ void JSONIOHandlerImpl::extendDataset( throw std::runtime_error( "[JSON] The specified location contains no valid dataset"); } - switch (stringToDatatype(j["datatype"].get())) + auto extent = parameters.extent; + auto datatype = stringToDatatype(j["datatype"].get()); + switch (datatype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - // @todo test complex resizing - auto complexExtent = parameters.extent; - complexExtent.push_back(2); - nlohmann::json newData = initializeNDArray(complexExtent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + extent.push_back(2); break; } default: - nlohmann::json newData = initializeNDArray(parameters.extent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + // nothing to do break; } + // TOML does not support nulls, so initialize with zero + nlohmann::json newData = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : datatype); + nlohmann::json &oldData = j["data"]; + mergeInto(newData, oldData); + j["data"] = newData; writable->written = true; } @@ -521,11 +598,7 @@ void JSONIOHandlerImpl::openFile( "Supplied directory is not valid: " + m_handler->directory); } - std::string name = parameter.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameter.name + m_originalExtension; auto file = std::get<0>(getPossiblyExisting(name)); @@ -844,7 +917,8 @@ void JSONIOHandlerImpl::readAttribute( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto name = removeSlashes(parameters.name); - auto &jsonLoc = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + auto const &jsonLoc = jsonContents["attributes"]; setAndGetFilePosition(writable); std::string error_msg("[JSON] No such attribute '"); if (!hasKey(jsonLoc, name)) @@ -919,7 +993,12 @@ void JSONIOHandlerImpl::listAttributes( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto filePosition = setAndGetFilePosition(writable); - auto &j = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + if (!jsonContents.contains("attributes")) + { + return; + } + auto const &j = jsonContents["attributes"]; for (auto it = j.begin(); it != j.end(); it++) { parameters.attributes->push_back(it.key()); @@ -932,14 +1011,16 @@ void JSONIOHandlerImpl::deregister( m_files.erase(writable); } -std::shared_ptr -JSONIOHandlerImpl::getFilehandle(File fileName, Access access) +auto JSONIOHandlerImpl::getFilehandle(File fileName, Access access) + -> std::tuple, std::istream *, std::ostream *> { VERIFY_ALWAYS( fileName.valid(), "[JSON] Tried opening a file that has been overwritten or deleted.") auto path = fullPath(std::move(fileName)); - auto fs = std::make_shared(); + auto fs = std::make_unique(); + std::istream *istream = nullptr; + std::ostream *ostream = nullptr; if (access::write(access)) { /* @@ -949,14 +1030,31 @@ JSONIOHandlerImpl::getFilehandle(File fileName, Access access) * equivalent, but the openPMD frontend exposes no reading * functionality in APPEND mode. */ - fs->open(path, std::ios_base::out | std::ios_base::trunc); + std::ios_base::openmode openmode = + std::ios_base::out | std::ios_base::trunc; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + ostream = + &(*fs << std::setprecision( + std::numeric_limits::digits10 + 1)); } else { - fs->open(path, std::ios_base::in); + std::ios_base::openmode openmode = std::ios_base::in; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + istream = + &(*fs >> + std::setprecision(std::numeric_limits::digits10 + 1)); } VERIFY(fs->good(), "[JSON] Failed opening a file '" + path + "'"); - return fs; + return std::make_tuple(std::move(fs), istream, ostream); } std::string JSONIOHandlerImpl::fullPath(File fileName) @@ -1048,26 +1146,6 @@ Extent JSONIOHandlerImpl::getMultiplicators(Extent const &extent) return res; } -nlohmann::json JSONIOHandlerImpl::initializeNDArray(Extent const &extent) -{ - // idea: begin from the innermost shale and copy the result into the - // outer shales - nlohmann::json accum; - nlohmann::json old; - auto *accum_ptr = &accum; - auto *old_ptr = &old; - for (auto it = extent.rbegin(); it != extent.rend(); it++) - { - std::swap(old_ptr, accum_ptr); - *accum_ptr = nlohmann::json{}; - for (Extent::value_type i = 0; i < *it; i++) - { - (*accum_ptr)[i] = *old_ptr; // copy boi - } - } - return *accum_ptr; -} - Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j) { Extent res; @@ -1106,7 +1184,7 @@ std::string JSONIOHandlerImpl::removeSlashes(std::string s) } template -bool JSONIOHandlerImpl::hasKey(nlohmann::json &j, KeyT &&key) +bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) { return j.find(std::forward(key)) != j.end(); } @@ -1166,9 +1244,19 @@ std::shared_ptr JSONIOHandlerImpl::obtainJsonContents(File file) return it->second; } // read from file - auto fh = getFilehandle(file, Access::READ_ONLY); + auto [fh, fh_with_precision, _] = getFilehandle(file, Access::READ_ONLY); + (void)_; std::shared_ptr res = std::make_shared(); - *fh >> *res; + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision >> *res; + break; + case FileFormat::Toml: + *res = + openPMD::json::tomlToJson(toml::parse(*fh_with_precision, *file)); + break; + } VERIFY(fh->good(), "[JSON] Failed reading from a file."); m_jsonVals.emplace(file, res); return res; @@ -1192,9 +1280,22 @@ void JSONIOHandlerImpl::putJsonContents( auto it = m_jsonVals.find(filename); if (it != m_jsonVals.end()) { - auto fh = getFilehandle(filename, Access::CREATE); + auto [fh, _, fh_with_precision] = + getFilehandle(filename, Access::CREATE); + (void)_; (*it->second)["platform_byte_widths"] = platformSpecifics(); - *fh << *it->second << std::endl; + + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision << *it->second << std::endl; + break; + case FileFormat::Toml: + *fh_with_precision << openPMD::json::jsonToToml(*it->second) + << std::endl; + break; + } + VERIFY(fh->good(), "[JSON] Failed writing data to disk.") m_jsonVals.erase(it); if (unsetDirty) @@ -1399,7 +1500,7 @@ void JSONIOHandlerImpl::AttributeWriter::call( template void JSONIOHandlerImpl::AttributeReader::call( - nlohmann::json &json, Parameter ¶meters) + nlohmann::json const &json, Parameter ¶meters) { JsonToCpp jtc; *parameters.resource = jtc(json); diff --git a/src/Series.cpp b/src/Series.cpp index 4a22330179..59a96efdb7 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2180,7 +2180,8 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) std::map const backendDescriptors{ {"hdf5", Format::HDF5}, {"adios2", Format::ADIOS2_BP}, - {"json", Format::JSON}}; + {"json", Format::JSON}, + {"toml", Format::TOML}}; std::string backend; getJsonOptionLowerCase(options, "backend", backend); if (!backend.empty()) diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index c04e672ae6..168cab7bf6 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -200,6 +200,7 @@ namespace return nlohmann::json(); // null } + // @todo maybe generalize error type throw error::BackendConfigSchema( currentPath, "Unexpected datatype in TOML configuration. This is probably a " @@ -215,7 +216,8 @@ namespace switch (val.type()) { case nlohmann::json::value_t::null: - return toml::value(); + throw error::BackendConfigSchema( + currentPath, "TOML does not support null values."); case nlohmann::json::value_t::object: { toml::value::table_type res; for (auto pair = val.begin(); pair != val.end(); ++pair) @@ -247,7 +249,7 @@ namespace case nlohmann::json::value_t::number_unsigned: return val.get(); case nlohmann::json::value_t::number_float: - return val.get(); + return (long double)val.get(); case nlohmann::json::value_t::binary: return val.get(); case nlohmann::json::value_t::discarded: @@ -501,7 +503,7 @@ std::optional asLowerCaseStringDynamic(nlohmann::json const &value) std::vector backendKeys() { - return {"adios2", "json", "hdf5"}; + return {"adios2", "json", "toml", "hdf5"}; } void warnGlobalUnusedOptions(TracingJSON const &config) diff --git a/src/binding/python/Access.cpp b/src/binding/python/Access.cpp index 8fcdcb73c7..3b1d2a4297 100644 --- a/src/binding/python/Access.cpp +++ b/src/binding/python/Access.cpp @@ -18,13 +18,9 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/IO/Access.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_Access(py::module &m) { diff --git a/src/binding/python/Attributable.cpp b/src/binding/python/Attributable.cpp index a18ae2c471..88dbb95cbc 100644 --- a/src/binding/python/Attributable.cpp +++ b/src/binding/python/Attributable.cpp @@ -22,11 +22,9 @@ #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attribute.hpp" -#include "openPMD/binding/python/Numpy.hpp" -#include -#include -#include +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Numpy.hpp" #include #include @@ -35,11 +33,7 @@ #include #include -namespace py = pybind11; -using namespace openPMD; - using PyAttributeKeys = std::vector; -// PYBIND11_MAKE_OPAQUE(PyAttributeKeys) bool setAttributeFromBufferInfo( Attributable &attr, std::string const &key, py::buffer &a) diff --git a/src/binding/python/BaseRecord.cpp b/src/binding/python/BaseRecord.cpp index e893b54174..7aeee2d38a 100644 --- a/src/binding/python/BaseRecord.cpp +++ b/src/binding/python/BaseRecord.cpp @@ -18,18 +18,14 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/backend/BaseRecord.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" -#include "openPMD/binding/python/UnitDimension.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/UnitDimension.hpp" void init_BaseRecord(py::module &m) { diff --git a/src/binding/python/BaseRecordComponent.cpp b/src/binding/python/BaseRecordComponent.cpp index 0a3a306e76..95be596560 100644 --- a/src/binding/python/BaseRecordComponent.cpp +++ b/src/binding/python/BaseRecordComponent.cpp @@ -18,19 +18,14 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include -#include - -#include "openPMD/Datatype.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/Datatype.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" #include -namespace py = pybind11; -using namespace openPMD; - void init_BaseRecordComponent(py::module &m) { py::class_(m, "Base_Record_Component") diff --git a/src/binding/python/ChunkInfo.cpp b/src/binding/python/ChunkInfo.cpp index 5edc29353b..d86a579905 100644 --- a/src/binding/python/ChunkInfo.cpp +++ b/src/binding/python/ChunkInfo.cpp @@ -18,17 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/ChunkInfo.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Chunk(py::module &m) { py::class_(m, "ChunkInfo") diff --git a/src/binding/python/Container.cpp b/src/binding/python/Container.cpp index 8adebc570d..ef0a194637 100644 --- a/src/binding/python/Container.cpp +++ b/src/binding/python/Container.cpp @@ -23,23 +23,9 @@ * * BSD-style license, see pybind11 LICENSE file. */ - -#include -#include -#include - -#include "openPMD/Iteration.hpp" -#include "openPMD/Mesh.hpp" -#include "openPMD/ParticlePatches.hpp" -#include "openPMD/ParticleSpecies.hpp" -#include "openPMD/Record.hpp" -#include "openPMD/Series.hpp" -#include "openPMD/backend/BaseRecord.hpp" -#include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/backend/Container.hpp" -#include "openPMD/backend/MeshRecordComponent.hpp" -#include "openPMD/backend/PatchRecord.hpp" -#include "openPMD/backend/PatchRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include #include @@ -47,9 +33,6 @@ #include #include -namespace py = pybind11; -using namespace openPMD; - namespace detail { /* based on std_bind.h in pybind11 @@ -156,27 +139,6 @@ bind_container(py::handle scope, std::string const &name, Args &&...args) } } // namespace detail -using PyIterationContainer = Series::IterationsContainer_t; -using PyMeshContainer = Container; -using PyPartContainer = Container; -using PyPatchContainer = Container; -using PyRecordContainer = Container; -using PyPatchRecordContainer = Container; -using PyRecordComponentContainer = Container; -using PyMeshRecordComponentContainer = Container; -using PyPatchRecordComponentContainer = Container; -using PyBaseRecordComponentContainer = Container; -PYBIND11_MAKE_OPAQUE(PyIterationContainer) -PYBIND11_MAKE_OPAQUE(PyMeshContainer) -PYBIND11_MAKE_OPAQUE(PyPartContainer) -PYBIND11_MAKE_OPAQUE(PyPatchContainer) -PYBIND11_MAKE_OPAQUE(PyRecordContainer) -PYBIND11_MAKE_OPAQUE(PyPatchRecordContainer) -PYBIND11_MAKE_OPAQUE(PyRecordComponentContainer) -PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) -PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) -PYBIND11_MAKE_OPAQUE(PyBaseRecordComponentContainer) - void init_Container(py::module &m) { ::detail::bind_container(m, "Iteration_Container"); diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index bdd35db956..a67d7f1221 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -18,17 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Dataset.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" #include -namespace py = pybind11; -using namespace openPMD; - void init_Dataset(py::module &m) { py::class_(m, "Dataset") diff --git a/src/binding/python/Datatype.cpp b/src/binding/python/Datatype.cpp index 6c5587725d..9d53fba5ca 100644 --- a/src/binding/python/Datatype.cpp +++ b/src/binding/python/Datatype.cpp @@ -18,14 +18,10 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Datatype.hpp" -#include "openPMD/binding/python/Numpy.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Numpy.hpp" void init_Datatype(py::module &m) { diff --git a/src/binding/python/Error.cpp b/src/binding/python/Error.cpp index df1b39a5b2..681398c579 100644 --- a/src/binding/python/Error.cpp +++ b/src/binding/python/Error.cpp @@ -1,9 +1,14 @@ +/* Copyright 2019-2023 The openPMD Community + * + * This header is used to centrally define classes that shall not violate the + * C++ one-definition-rule (ODR) for various Python translation units. + * + * Authors: Franz Poeschel, Axel Huebl + * License: LGPL-3.0-or-later + */ #include "openPMD/Error.hpp" -#include - -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_Error(py::module &m) { diff --git a/src/binding/python/Helper.cpp b/src/binding/python/Helper.cpp index 9b07a37f70..e0aef84e0d 100644 --- a/src/binding/python/Helper.cpp +++ b/src/binding/python/Helper.cpp @@ -18,20 +18,16 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Series.hpp" #include "openPMD/cli/ls.hpp" #include "openPMD/helper/list_series.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Helper(py::module &m) { m.def( diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index b54c81c626..22e51aa974 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -18,18 +18,14 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Iteration.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Iteration(py::module &m) { py::class_(m, "Iteration") diff --git a/src/binding/python/IterationEncoding.cpp b/src/binding/python/IterationEncoding.cpp index 479ef65555..ef412e2639 100644 --- a/src/binding/python/IterationEncoding.cpp +++ b/src/binding/python/IterationEncoding.cpp @@ -18,13 +18,9 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/IterationEncoding.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_IterationEncoding(py::module &m) { diff --git a/src/binding/python/Mesh.cpp b/src/binding/python/Mesh.cpp index 8b6da30e64..e883e5e496 100644 --- a/src/binding/python/Mesh.cpp +++ b/src/binding/python/Mesh.cpp @@ -18,21 +18,17 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Mesh.hpp" #include "openPMD/backend/BaseRecord.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/binding/python/UnitDimension.hpp" #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Mesh(py::module &m) { py::class_ > cl(m, "Mesh"); diff --git a/src/binding/python/MeshRecordComponent.cpp b/src/binding/python/MeshRecordComponent.cpp index 98e602bbcb..120fe00da2 100644 --- a/src/binding/python/MeshRecordComponent.cpp +++ b/src/binding/python/MeshRecordComponent.cpp @@ -18,20 +18,17 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include +#include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" -#include "openPMD/backend/MeshRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include #include -namespace py = pybind11; -using namespace openPMD; - void init_MeshRecordComponent(py::module &m) { py::class_ cl( diff --git a/src/binding/python/ParticlePatches.cpp b/src/binding/python/ParticlePatches.cpp index 28deeab1b5..326162191d 100644 --- a/src/binding/python/ParticlePatches.cpp +++ b/src/binding/python/ParticlePatches.cpp @@ -18,17 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/ParticlePatches.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/PatchRecord.hpp" -#include +#include "openPMD/binding/python/Common.hpp" -namespace py = pybind11; -using namespace openPMD; +#include void init_ParticlePatches(py::module &m) { diff --git a/src/binding/python/ParticleSpecies.cpp b/src/binding/python/ParticleSpecies.cpp index 349081ea8e..349ea8c689 100644 --- a/src/binding/python/ParticleSpecies.cpp +++ b/src/binding/python/ParticleSpecies.cpp @@ -18,22 +18,18 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/ParticleSpecies.hpp" #include "openPMD/Record.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/Container.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include #include #include -namespace py = pybind11; -using namespace openPMD; - void init_ParticleSpecies(py::module &m) { py::class_ > cl(m, "ParticleSpecies"); diff --git a/src/binding/python/PatchRecord.cpp b/src/binding/python/PatchRecord.cpp index 7d01add8ea..aaef39546c 100644 --- a/src/binding/python/PatchRecord.cpp +++ b/src/binding/python/PatchRecord.cpp @@ -18,16 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include +#include "openPMD/backend/PatchRecord.hpp" #include "openPMD/backend/BaseRecord.hpp" -#include "openPMD/backend/PatchRecord.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" -#include "openPMD/binding/python/UnitDimension.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/UnitDimension.hpp" void init_PatchRecord(py::module &m) { diff --git a/src/binding/python/PatchRecordComponent.cpp b/src/binding/python/PatchRecordComponent.cpp index bf6a687b04..e668f7a1d9 100644 --- a/src/binding/python/PatchRecordComponent.cpp +++ b/src/binding/python/PatchRecordComponent.cpp @@ -18,16 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include +#include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" -#include "openPMD/backend/PatchRecordComponent.hpp" -#include "openPMD/binding/python/Numpy.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Numpy.hpp" namespace { diff --git a/src/binding/python/Record.cpp b/src/binding/python/Record.cpp index ee96e304e6..d97641bbce 100644 --- a/src/binding/python/Record.cpp +++ b/src/binding/python/Record.cpp @@ -18,21 +18,17 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/Record.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/backend/BaseRecord.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/binding/python/UnitDimension.hpp" #include #include -namespace py = pybind11; -using namespace openPMD; - void init_Record(py::module &m) { py::class_ > cl(m, "Record"); diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 32e1e0fb0d..ee8bb7d3bd 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -18,15 +18,13 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include -#include - +#include "openPMD/RecordComponent.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" -#include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" + +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" #include "openPMD/binding/python/Pickle.hpp" @@ -42,9 +40,6 @@ #include #include -namespace py = pybind11; -using namespace openPMD; - /** Convert a py::tuple of py::slices to Offset & Extent * * https://docs.scipy.org/doc/numpy-1.15.0/reference/arrays.indexing.html diff --git a/src/binding/python/UnitDimension.cpp b/src/binding/python/UnitDimension.cpp index 72074cde78..6e46a6cfcf 100644 --- a/src/binding/python/UnitDimension.cpp +++ b/src/binding/python/UnitDimension.cpp @@ -18,13 +18,9 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/UnitDimension.hpp" -namespace py = pybind11; -using namespace openPMD; +#include "openPMD/binding/python/Common.hpp" void init_UnitDimension(py::module &m) { diff --git a/src/binding/python/openPMD.cpp b/src/binding/python/openPMD.cpp index 5133c66d21..28b56b7d9c 100644 --- a/src/binding/python/openPMD.cpp +++ b/src/binding/python/openPMD.cpp @@ -18,18 +18,15 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ -#include -#include - #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include "openPMD/binding/python/Common.hpp" + #include #include #include -namespace py = pybind11; - // forward declarations of exposed classes void init_Access(py::module &); void init_Attributable(py::module &); diff --git a/src/binding/python/openpmd_api/DaskArray.py b/src/binding/python/openpmd_api/DaskArray.py index 8c6fc3001b..709b668be8 100644 --- a/src/binding/python/openpmd_api/DaskArray.py +++ b/src/binding/python/openpmd_api/DaskArray.py @@ -1,8 +1,8 @@ """ This file is part of the openPMD-api. -Copyright 2021 openPMD contributors -Authors: Axel Huebl +Copyright 2021-2023 openPMD contributors +Authors: Axel Huebl, Pawel Ordyna License: LGPLv3+ """ import math @@ -49,7 +49,7 @@ def __getitem__(self, slices): return data -def record_component_to_daskarray(record_component): +def record_component_to_daskarray(record_component, chunks=None): """ Load a RecordComponent into a Dask.array. @@ -57,6 +57,10 @@ def record_component_to_daskarray(record_component): ---------- record_component : openpmd_api.Record_Component A record component class in openPMD-api. + chunks : chunks parameter to pass to dask.array.from_array. + See dask documentation for more details. + When set to None (default) the chunking will be automaticaly + determined based on record_component.available_chunks(). Returns ------- @@ -84,54 +88,56 @@ def record_component_to_daskarray(record_component): if not found_dask: raise ImportError("dask NOT found. Install dask for Dask DataFrame " "support.") - - # get optimal chunks - chunks = record_component.available_chunks() - - # sort and prepare the chunks for Dask's array API - # https://docs.dask.org/en/latest/array-chunks.html - # https://docs.dask.org/en/latest/array-api.html?highlight=from_array#other-functions - # sorted and unique - offsets_per_dim = list(map(list, zip(*[chunk.offset for chunk in chunks]))) - offsets_sorted_unique_per_dim = [sorted(set(o)) for o in offsets_per_dim] - - # print("offsets_sorted_unique_per_dim=", - # list(offsets_sorted_unique_per_dim)) - - # case 1: PIConGPU static load balancing (works with Dask assumptions, - # chunk option no. 3) - # all chunks in the same column have the same column width although - # individual columns have different widths - # case 2: AMReX boxes - # all chunks are multiple of a common block size, offsets are a multiple - # of a common blocksize - # problem: too limited description in Dask - # https://github.com/dask/dask/issues/7475 - # work-around: create smaller chunks (this incurs a read cost) by forcing - # into case 1 - # (this can lead to larger blocks than using the gcd of the - # extents aka AMReX block size) - common_chunk_widths_per_dim = list() - for d, offsets_in_dim in enumerate(offsets_sorted_unique_per_dim): - # print("d=", d, offsets_in_dim, record_component.shape[d]) - offsets_in_dim_arr = np.array(offsets_in_dim) - # note: this is in the right order of rows/columns, contrary to a - # sorted extent list from chunks - extents_in_dim = np.zeros_like(offsets_in_dim_arr) - extents_in_dim[:-1] = offsets_in_dim_arr[1:] - extents_in_dim[-1] = record_component.shape[d] - if len(extents_in_dim) > 1: - extents_in_dim[:-1] -= offsets_in_dim_arr[:-1] - extents_in_dim[-1] -= offsets_in_dim_arr[-1] - # print("extents_in_dim=", extents_in_dim) - common_chunk_widths_per_dim.append(tuple(extents_in_dim)) - - common_chunk_widths_per_dim = tuple(common_chunk_widths_per_dim) - # print("common_chunk_widths_per_dim=", common_chunk_widths_per_dim) + if chunks is None: + # get optimal chunks + chunks = record_component.available_chunks() + + # sort and prepare the chunks for Dask's array API + # https://docs.dask.org/en/latest/array-chunks.html + # https://docs.dask.org/en/latest/array-api.html?highlight=from_array#other-functions + # sorted and unique + offsets_per_dim = list( + map(list, zip(*[chunk.offset for chunk in chunks]))) + offsets_sorted_unique_per_dim = [ + sorted(set(o)) for o in offsets_per_dim] + + # print("offsets_sorted_unique_per_dim=", + # list(offsets_sorted_unique_per_dim)) + + # case 1: PIConGPU static load balancing (works with Dask assumptions, + # chunk option no. 3) + # all chunks in the same column have the same column width although + # individual columns have different widths + # case 2: AMReX boxes + # all chunks are multiple of a common block size, offsets + # are a multiple of a common blocksize + # problem: too limited description in Dask + # https://github.com/dask/dask/issues/7475 + # work-around: create smaller chunks (this incurs a read cost) + # by forcing into case 1 + # (this can lead to larger blocks than using + # the gcd of the extents aka AMReX block size) + common_chunk_widths_per_dim = list() + for d, offsets_in_dim in enumerate(offsets_sorted_unique_per_dim): + # print("d=", d, offsets_in_dim, record_component.shape[d]) + offsets_in_dim_arr = np.array(offsets_in_dim) + # note: this is in the right order of rows/columns, contrary to a + # sorted extent list from chunks + extents_in_dim = np.zeros_like(offsets_in_dim_arr) + extents_in_dim[:-1] = offsets_in_dim_arr[1:] + extents_in_dim[-1] = record_component.shape[d] + if len(extents_in_dim) > 1: + extents_in_dim[:-1] -= offsets_in_dim_arr[:-1] + extents_in_dim[-1] -= offsets_in_dim_arr[-1] + # print("extents_in_dim=", extents_in_dim) + common_chunk_widths_per_dim.append(tuple(extents_in_dim)) + + chunks = tuple(common_chunk_widths_per_dim) + # print("chunks=", chunks) da = from_array( DaskRecordComponent(record_component), - chunks=common_chunk_widths_per_dim, + chunks=chunks, # name=None, asarray=True, fancy=False, diff --git a/src/config.cpp b/src/config.cpp index a44925287a..89a824500c 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -28,20 +28,31 @@ #include #include +// @todo add TOML here std::map openPMD::getVariants() { + // clang-format off return std::map{ {"mpi", bool(openPMD_HAVE_MPI)}, {"json", true}, +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 + {"toml", true}, +#endif {"hdf5", bool(openPMD_HAVE_HDF5)}, {"adios1", false}, {"adios2", bool(openPMD_HAVE_ADIOS2)}}; + // clang-format on } std::vector openPMD::getFileExtensions() { std::vector fext; fext.emplace_back("json"); +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 + fext.emplace_back("toml"); +#endif #if openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 224596864e..d660e29ec4 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1051,7 +1051,7 @@ TEST_CASE("no_file_ending", "[core]") Access::CREATE, R"({"backend": "json"})"); } - REQUIRE(auxiliary::file_exists("../samples/no_extension_specified.json")); + REQUIRE(auxiliary::file_exists("../samples/no_extension_specified")); } TEST_CASE("backend_via_json", "[core]") diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 11297c7595..962ca636aa 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -81,20 +81,20 @@ std::vector testedFileExtensions() allExtensions.begin(), allExtensions.end(), []([[maybe_unused]] std::string const &ext) { -#if openPMD_HAVE_ADIOS2 -#define HAS_ADIOS_2_9 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) -#if HAS_ADIOS_2_9 +#if openPMD_HAS_ADIOS_2_9 // sst and ssc need a receiver for testing // bp5 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp5"; + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests + return ext == "sst" || ext == "ssc" || ext == "bp5" || + ext == "toml"; #else + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests // sst and ssc need a receiver for testing // bp4 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp4"; -#endif -#undef HAS_ADIOS_2_9 -#else - return false; + return ext == "sst" || ext == "ssc" || ext == "bp4" || + ext == "toml"; #endif }); return {allExtensions.begin(), newEnd}; @@ -1231,7 +1231,7 @@ TEST_CASE("particle_patches", "[serial]") inline void dtype_test(const std::string &backend) { - bool test_long_double = (backend != "json") || sizeof(long double) <= 8; + bool test_long_double = backend != "json" && backend != "toml"; bool test_long_long = (backend != "json") || sizeof(long long) <= 8; { Series s = Series("../samples/dtype_test." + backend, Access::CREATE); @@ -1447,7 +1447,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); REQUIRE(s.getAttribute("int").dtype == Datatype::INT); REQUIRE(s.getAttribute("long").dtype == Datatype::LONG); - REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + } REQUIRE(s.getAttribute("ushort").dtype == Datatype::USHORT); REQUIRE(s.getAttribute("uint").dtype == Datatype::UINT); REQUIRE(s.getAttribute("ulong").dtype == Datatype::ULONG); @@ -1459,7 +1462,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("vecShort").dtype == Datatype::VEC_SHORT); REQUIRE(s.getAttribute("vecInt").dtype == Datatype::VEC_INT); REQUIRE(s.getAttribute("vecLong").dtype == Datatype::VEC_LONG); - REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + } REQUIRE(s.getAttribute("vecUShort").dtype == Datatype::VEC_USHORT); REQUIRE(s.getAttribute("vecUInt").dtype == Datatype::VEC_UINT); REQUIRE(s.getAttribute("vecULong").dtype == Datatype::VEC_ULONG); @@ -1508,6 +1514,15 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + dtype_test("toml"); + } } inline void write_test(const std::string &backend) @@ -2106,6 +2121,15 @@ TEST_CASE("fileBased_write_test", "[serial]") { fileBased_write_test(t); } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + fileBased_write_test("toml"); + } } inline void sample_write_thetaMode(std::string file_ending) @@ -7405,4 +7429,13 @@ TEST_CASE("groupbased_read_write", "[serial]") } } } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + groupbased_read_write("toml"); + } } diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 0aa71cf7a8..a510098c8d 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -359,7 +359,8 @@ def attributeRoundTrip(self, file_ending): # c_types self.assertEqual(series.get_attribute("byte_c"), 30) self.assertEqual(series.get_attribute("ubyte_c"), 50) - if file_ending != "json": # TODO: returns [100] instead of 100 in json + # TODO: returns [100] instead of 100 in json/toml + if file_ending != "json" and file_ending != "toml": self.assertEqual(chr(series.get_attribute("char_c")), 'd') self.assertEqual(series.get_attribute("int16_c"), 2) self.assertEqual(series.get_attribute("int32_c"), 3) @@ -1824,7 +1825,7 @@ def testIterator(self): self.makeIteratorRoundTrip(b, backend_filesupport[b]) def makeAvailableChunksRoundTrip(self, ext): - if ext == "h5": + if ext == "h5" or ext == "toml": return name = "../samples/available_chunks_python." + ext write = io.Series(