diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e950645 --- /dev/null +++ b/.gitignore @@ -0,0 +1,70 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Sphinx documentation +docs/build/ + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv3/ +venv-doc/ + +# mypy +.mypy_cache/ + +# IDE +.idea + +# Archive +obsolete/ + +# Project specific +messages/ +example/ +benchmarks/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8992dbe --- /dev/null +++ b/.travis.yml @@ -0,0 +1,40 @@ +matrix: + include: + - language: python + python: + - "3.5" + - "3.6" + addons: + apt: + packages: + before_script: + - cd py + install: + - pip3 install -e .[dev] + - pip3 install coveralls + script: + - python3 precommit.py + - coveralls + - language: cpp + os: linux + dist: xenial + compiler: + - clang + - gcc + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.8 + packages: + - g++-6 + - clang-3.8 + install: + - make get-deps + - "[ $CXX = g++ ] && export CXX=g++-6 || true" + - "[ $CXX = clang++ ] && export CXX=clang++-3.8 || true" + before_script: + - cd cpp + script: + - make + - make test \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..43bb01b --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,3 @@ +1.0.0 +===== +* Initial version \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..706450c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Parquery AG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index fab3032..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# persipubsub -TODO diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..b794c08 --- /dev/null +++ b/README.rst @@ -0,0 +1,367 @@ +persipubsub +=========== + +.. image:: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :target: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :alt: Build Status + +.. image:: https://coveralls.io/repos/github/Parquery/persipubsub/badge.svg?branch=master + :target: https://coveralls.io/github/Parquery/persipubsub?branch=master + :alt: Coverage + +.. image:: https://readthedocs.org/projects/persipubsub/badge/?version=latest + :target: https://persipubsub.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://badge.fury.io/py/persipubsub.svg + :target: https://badge.fury.io/py/persipubsub + :alt: PyPI - version + +.. image:: https://img.shields.io/pypi/pyversions/persipubsub.svg + :alt: PyPI - Python Version + +.. image:: https://badges.frapsoft.com/os/mit/mit.png?v=103 + :target: https://opensource.org/licenses/mit-license.php + :alt: MIT License + +``persipubsub`` implements a persistent, thread-safe and process-safe queue for +inter-process communication, based on `lmdb `_. + +Primarily, we used `zeromq `_ for inter-process +communication with a slight improvement through `persizmq +`_. This still did not fulfill the level +of persistence we wanted. + +Our motivation was to replace our previous library with a one which is +similarly easy to setup and to use. Additionally, it should make it possible to +send `protobuf `_ messages +(bytes) thread-safely and process-safely from many publishers to many +subscribers. + +Besides basic publisher and subscriber classes the library offers control +methods for easy deployment from a config JSON file and maintenance in case +needed. + +Related projects +================ + +persist-queue +------------- + +* The library offers not all functionality expected from a queue. It has put + and get function which are basically only push and pop. Therefore ``front`` + functionality is missing. In consequence neither can the queue have multiple + subscribers nor can be guaranteed that no data is lost when a thread fails. +* All messages in queues are serialized by ``pickle`` which was for us the main + reason not to use this library. + +Kafka +----- + +* Hence we only need Inter Process Communication, the TCP ability of `Kafka + `_ is an unnecessary overhead. +* Integration of ``Kafka`` written in Scala and Java in our C++/Python/Go + codebase is challenging. +* Hard to setup and maintain ``Kafka``. +* broker system eventually a bottleneck. + +RabbitMQ +-------- + +* Hence we only need Inter Process Communication, the TCP ability of `RabbitMQ + `_ is an unnecessary overhead. +* broker system eventually a bottleneck. +* ``RabbitMQ`` is less scalable than ``Kafka``, but + is supported officially by more languages. + +zeromq persistence pattern +-------------------------- + +* `Titanic `_ is the only persistence + pattern of `zeromq `_ which is also a broker system. + This takes away the purpose and advantage of ``zeromq`` to be a + lightweight library which requires no broker. + +Usage +===== + +The usage of the library consists of two steps: deployment and runtime + +Python +====== + +Environment +----------- + +To improve the accessibility of the library, an environment class lets you +create and initialize any ``persipubsub`` component which you need in +deployment or runtime step. + +.. warning:: + + Only one environment of each queue per process allowed and it's forbidden to + fork environment or any child components to multiple processes. + In that case, persipubsub is multi-threading and multi-process safe. + If multiple environments of the same queue are active on the same process, + or environment is forked to multiple processes the lock is broken and + correctness can't be guaranteed. + +Initialize environment +^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + +Deployment +---------- + +In the deployment stage the library sets up the queue structure with the control. + +Control +^^^^^^^ + +A control unit to initialize and maintain queues. + +.. note:: + + The high water marks are limits for the queue. The message is deleted in + case that it reaches the timeout. In the other case of an overflow one + of two strategies is used to prune half of the queue. The choice is between + prune_first, which deletes the oldest messages, and prune_last, which + deletes the latest messages. + +Initialize queue +"""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + import persipubsub.queue + + env = persipubsub.environment.new_environment(path="/home/user/new-queue/") + + # Initialize a queue with default values. + control = env.new_control() + # Or define all optional parameters of the queue. + hwm = persipubsub.queue._HighWaterMark() + strategy = persipubsub.queue._Strategy.prune_first + control = env.new_control(subscriber_ids=["sub1", "sub2"], + high_watermark=hwm, + strategy=strategy) + +Prune all dangling messages +""""""""""""""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-dangling-messages/") + control = env.new_control() + + control.prune_dangling_messages() + +Clear all messages +"""""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-subscribers-and-messages/") + control = env.new_control() + + control.clear_all_subscribers() + + +Runtime +------- + +During runtime only publisher and subscriber are needed. + +.. note:: + + Control can be optionally be used for pruning although the queues prune + itself on a regular basis when high water mark is reached. The high water + mark includes a timeout, maximum number of messages and the maximum bytes + size of the queue. + +Publisher +^^^^^^^^^ + +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the publisher can +be initialized as following. + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + + pub = env.new_publisher() + +Send a message +"""""""""""""" + +.. code-block:: python + + msg = "Hello there!".encode('utf-8') + pub.send(msg=msg) + + # Subscribers have now a message in the queue. + +Send many messages at once +"""""""""""""""""""""""""" + +.. code-block:: python + + msgs = ["What's up?".encode('utf-8'), + "Do you like the README?".encode('utf-8')] + pub.send_many(msgs=msgs) + + # Both messages are now available for the subscribers. Note that the order + # of the messages are not necessarily kept. + +Subscriber +^^^^^^^^^^ + +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the subscriber can +be initialized as following. + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + + sub = env.new_subscriber(identifier="sub") + +Receive a message +""""""""""""""""" + +.. code-block:: python + + # One message in queue + with sub.receive() as msg: + # do something with the message + print(msg) # b'Hello there!' + + # This subscriber's queue is now empty + +Catch up with latest message +"""""""""""""""""""""""""""" + +Can be used in the case when a particular subscriber cares only about the very +last message. The messages are not popped for other subscribers. + +.. note:: + If you want to store only the latest message for all subscribers, then use + high water mark max_msgs_num = 1. + + +.. code-block:: python + + # Many outdated messages in queue + + with sub.receive_to_top() as msg: + # do something with the latest message + + # This subscriber's queue is now empty. + +Documentation +============= + +The documentation is available on `readthedocs +`_. + +Installation +============ + +* Install persipubsub with pip: + +.. code-block:: bash + + pip3 install persipubsub + +Development +=========== + +* Check out the repository. + +* In the repository root, create the virtual environment: + +.. code-block:: bash + + python3 -m venv venv3 + +* Activate the virtual environment: + +.. code-block:: bash + + source venv3/bin/activate + +* Install the development dependencies: + +.. code-block:: bash + + pip3 install -e .[dev] + +We use tox for testing and packaging the distribution. Assuming that the virtual +environment has been activated and the development dependencies have been +installed, run: + +.. code-block:: bash + + tox + +Pre-commit Checks +----------------- + +We provide a set of pre-commit checks that lint and check code for formatting. + +Namely, we use: + +* `yapf `_ to check the formatting. +* The style of the docstrings is checked with `pydocstyle `_. +* Static type analysis is performed with `mypy `_. +* `isort `_ to sort your imports for you. +* Various linter checks are done with `pylint `_. +* Doctests are executed using the Python `doctest module `_. +* `pyicontract-lint `_ lints contracts + in Python code defined with `icontract library `_. +* `twine `_ to check the README for invalid markup + which prevents it from rendering correctly on PyPI. + +Run the pre-commit checks locally from an activated virtual environment with +development dependencies: + +.. code-block:: bash + + ./precommit.py + +* The pre-commit script can also automatically format the code: + +.. code-block:: bash + + ./precommit.py --overwrite + +Versioning +========== + +We follow `Semantic Versioning `_. +The version X.Y.Z indicates: + +* X is the major version (backward-incompatible), +* Y is the minor version (backward-compatible), and +* Z is the patch version (backward-compatible bug fix). \ No newline at end of file diff --git a/cpp/.gitignore b/cpp/.gitignore new file mode 100644 index 0000000..05d2a2a --- /dev/null +++ b/cpp/.gitignore @@ -0,0 +1,22 @@ +# Distribution / packaging +cmake-build-debug/ +TODO + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv3/ +venv-doc/ + +# IDE +.idea + +# Archive +obsolete/ + +# Project specific \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..17aab6e --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.12) +project(persipubsub) + +set(CMAKE_CXX_STANDARD 17) + +include_directories(include) + +enable_testing() +add_subdirectory(src) +add_subdirectory(test) +add_test(NAME myTest COMMAND Test) \ No newline at end of file diff --git a/cpp/include/filesystem.h b/cpp/include/filesystem.h new file mode 100755 index 0000000..039266c --- /dev/null +++ b/cpp/include/filesystem.h @@ -0,0 +1,153 @@ +// Copyright (c) 2016 Parquery AG. All rights reserved. +// Created by mristin on 10/12/16. + +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +namespace pqry { +namespace filesystem { + +/** + * Simulates Unix ls -lt + * @param directory to list + * @return absolute paths sorted by modification time + */ +std::vector ls_lt(const boost::filesystem::path& directory); + +/** + * @param directory to be listed + * @return list of absolute paths to files contained in the directory + */ +std::vector listdir(const boost::filesystem::path& directory); + +/** + * @param path to the file + * @return modified time in nanoseconds since epoch + */ +int64_t modified_time(const boost::filesystem::path& path); + +/** + * simulates `mkdtemp` command + * @return path to the temporary directory + */ +boost::filesystem::path mkdtemp(); + +/** + * Reads a file to the string + * @param path to the file + * @return content of the file + */ +std::string read(const boost::filesystem::path& path); + +/** + * Writes the whole text to the file + * @param path to the file + * @param text to be written + */ +void write(const boost::filesystem::path& path, const std::string& text); + +/** + * Writes the `bytes` to the file + * @param path to write to + * @param bytes that are to be written + * @param size of the `bytes` + */ +void write(const boost::filesystem::path& path, const char* bytes, int size); + +/** + * Copies a source directory with all contents into a new target directory recursively + * @param source directory that gets copied + * @param dest directory where source gets copied to. Will be created if non-existent + */ +void copy_directory(const boost::filesystem::path& source, const boost::filesystem::path& dest); + +/** + * Unzips an archive. + * @param path to the archive + * @param dest_dir destination where the archive will be unzipped. Expects it to exist + */ +void unzip_archive(const boost::filesystem::path& path, const boost::filesystem::path& dest_dir); + +/** + * Uses POSIX flock() for named mutexes. Use an instance of this class to prevent multiple processes running at the + * same time. + * + * Mind that the instance keeps a file descriptor open during its lifetime. + * The implementation is crash-safe, i.e. if the program crashes, the system automatically releases the lock. + * Unlike boost::interprocess::file_lock, we do not have to create the file manually first. + * boost::interprocess:named_mutex did not work on Linux with boost 1.62 (marko tested it manually). + */ +class LockFileGuard { + public: + LockFileGuard() : locked_(false) {} + + /** + * Creates the lock file and writes the given PID to it + * @param lock_file to use for locking + * @param pid of the process + * @return true if the file could be successfully locked. + */ + bool lock(const boost::filesystem::path& lock_file, int pid); + + void unlock(); + + virtual ~LockFileGuard(); + + private: + bool locked_; + boost::filesystem::path lock_file_; + int fid_; +}; + +/** + * Wait for file to exist. Panics if timeout exceeded (0 means no timeout). + * @param path to the file + * @param timeout in seconds + */ +void wait_for_file(const boost::filesystem::path& path, unsigned int timeout); + +/** + * Expands path using wordexp + * @param path to be expanded + * @return expanded path + */ +boost::filesystem::path expand_path(const boost::filesystem::path& path); + +/** + * Temporary scope file, created from a path with a random suffix and ".tmp". + * If the file exists at the destruction, it will be deleted. + * All errors will be ignored in the destructor. + */ +class NamedTempfile { + public: + explicit NamedTempfile(const boost::filesystem::path& path); + + /** + * Deletes the temporary file if it was not renamed before. + */ + ~NamedTempfile(); + + const boost::filesystem::path& path() const { return tmp_pth_; } + + /** + * Renames the temporary file to the path passed in at the constructor. + */ + void rename(); + + private: + const boost::filesystem::path pth_; + const boost::filesystem::path tmp_pth_; + bool renamed_; +}; + +} // namespace filesystem +} // namespace pqry diff --git a/cpp/include/lmdb++.h b/cpp/include/lmdb++.h new file mode 100644 index 0000000..ab75f8c --- /dev/null +++ b/cpp/include/lmdb++.h @@ -0,0 +1,1913 @@ +/* This is free and unencumbered software released into the public domain. */ + +#ifndef LMDBXX_H +#define LMDBXX_H + +/** + * - C++11 wrapper for LMDB. + * + * @author Arto Bendiken + * @see https://sourceforge.net/projects/lmdbxx/ + */ + +#ifndef __cplusplus +#error " requires a C++ compiler" +#endif + +#if __cplusplus < 201103L +#if !defined(_MSC_VER) || _MSC_VER < 1900 +#error " requires a C++11 compiler (CXXFLAGS='-std=c++11')" +#endif // _MSC_VER check +#endif + +//////////////////////////////////////////////////////////////////////////////// + +#include /* for MDB_*, mdb_*() */ + +#ifdef LMDBXX_DEBUG +#include /* for assert() */ +#endif +#include /* for std::size_t */ +#include /* for std::snprintf() */ +#include /* for std::strlen() */ +#include /* for std::runtime_error */ +#include /* for std::string */ +#include /* for std::is_pod<> */ + +namespace lmdb { + using mode = mdb_mode_t; +} + +//////////////////////////////////////////////////////////////////////////////// +/* Error Handling */ + +namespace lmdb { + class error; + class logic_error; + class fatal_error; + class runtime_error; + class key_exist_error; + class not_found_error; + class corrupted_error; + class panic_error; + class version_mismatch_error; + class map_full_error; + class bad_dbi_error; +} + +/** + * Base class for LMDB exception conditions. + * + * @see http://symas.com/mdb/doc/group__errors.html + */ +class lmdb::error : public std::runtime_error { +protected: + const int _code; + +public: + /** + * Throws an error based on the given LMDB return code. + */ + [[noreturn]] static inline void raise(const char* origin, int rc); + + /** + * Constructor. + */ + error(const char* const origin, + const int rc) noexcept + : runtime_error{origin}, + _code{rc} {} + + /** + * Returns the underlying LMDB error code. + */ + int code() const noexcept { + return _code; + } + + /** + * Returns the origin of the LMDB error. + */ + const char* origin() const noexcept { + return runtime_error::what(); + } + + /** + * Returns the underlying LMDB error code. + */ + virtual const char* what() const noexcept { + static thread_local char buffer[1024]; + std::snprintf(buffer, sizeof(buffer), + "%s: %s", origin(), ::mdb_strerror(code())); + return buffer; + } +}; + +/** + * Base class for logic error conditions. + */ +class lmdb::logic_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Base class for fatal error conditions. + */ +class lmdb::fatal_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Base class for runtime error conditions. + */ +class lmdb::runtime_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Exception class for `MDB_KEYEXIST` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga05dc5bbcc7da81a7345bd8676e8e0e3b + */ +class lmdb::key_exist_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_NOTFOUND` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gabeb52e4c4be21b329e31c4add1b71926 + */ +class lmdb::not_found_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_CORRUPTED` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gaf8148bf1b85f58e264e57194bafb03ef + */ +class lmdb::corrupted_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_PANIC` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gae37b9aedcb3767faba3de8c1cf6d3473 + */ +class lmdb::panic_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_VERSION_MISMATCH` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga909b2db047fa90fb0d37a78f86a6f99b + */ +class lmdb::version_mismatch_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_MAP_FULL` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga0a83370402a060c9175100d4bbfb9f25 + */ +class lmdb::map_full_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_BAD_DBI` errors. + * + * @since 0.9.14 (2014/09/20) + * @see http://symas.com/mdb/doc/group__errors.html#gab4c82e050391b60a18a5df08d22a7083 + */ +class lmdb::bad_dbi_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +inline void +lmdb::error::raise(const char* const origin, + const int rc) { + switch (rc) { + case MDB_KEYEXIST: throw key_exist_error{origin, rc}; + case MDB_NOTFOUND: throw not_found_error{origin, rc}; + case MDB_CORRUPTED: throw corrupted_error{origin, rc}; + case MDB_PANIC: throw panic_error{origin, rc}; + case MDB_VERSION_MISMATCH: throw version_mismatch_error{origin, rc}; + case MDB_MAP_FULL: throw map_full_error{origin, rc}; +#ifdef MDB_BAD_DBI + case MDB_BAD_DBI: throw bad_dbi_error{origin, rc}; +#endif + default: throw lmdb::runtime_error{origin, rc}; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Metadata */ + +namespace lmdb { + // TODO: mdb_version() + // TODO: mdb_strerror() +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Environment */ + +namespace lmdb { + static inline void env_create(MDB_env** env); + static inline void env_open(MDB_env* env, + const char* path, unsigned int flags, mode mode); +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + static inline void env_copy(MDB_env* env, const char* path, unsigned int flags); + static inline void env_copy_fd(MDB_env* env, mdb_filehandle_t fd, unsigned int flags); +#else + static inline void env_copy(MDB_env* env, const char* path); + static inline void env_copy_fd(MDB_env* env, mdb_filehandle_t fd); +#endif + static inline void env_stat(MDB_env* env, MDB_stat* stat); + static inline void env_info(MDB_env* env, MDB_envinfo* stat); + static inline void env_sync(MDB_env* env, bool force); + static inline void env_close(MDB_env* env) noexcept; + static inline void env_set_flags(MDB_env* env, unsigned int flags, bool onoff); + static inline void env_get_flags(MDB_env* env, unsigned int* flags); + static inline void env_get_path(MDB_env* env, const char** path); + static inline void env_get_fd(MDB_env* env, mdb_filehandle_t* fd); + static inline void env_set_mapsize(MDB_env* env, std::size_t size); + static inline void env_set_max_readers(MDB_env* env, unsigned int count); + static inline void env_get_max_readers(MDB_env* env, unsigned int* count); + static inline void env_set_max_dbs(MDB_env* env, MDB_dbi count); + static inline unsigned int env_get_max_keysize(MDB_env* env); +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) + static inline void env_set_userctx(MDB_env* env, void* ctx); + static inline void* env_get_userctx(MDB_env* env); +#endif + // TODO: mdb_env_set_assert() + // TODO: mdb_reader_list() + // TODO: mdb_reader_check() +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaad6be3d8dcd4ea01f8df436f41d158d4 + */ +static inline void +lmdb::env_create(MDB_env** env) { + const int rc = ::mdb_env_create(env); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_create", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga32a193c6bf4d7d5c5d579e71f22e9340 + */ +static inline void +lmdb::env_open(MDB_env* const env, + const char* const path, + const unsigned int flags, + const mode mode) { + const int rc = ::mdb_env_open(env, path, flags, mode); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_open", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga3bf50d7793b36aaddf6b481a44e24244 + * @see http://symas.com/mdb/doc/group__mdb.html#ga5d51d6130325f7353db0955dbedbc378 + */ +static inline void +lmdb::env_copy(MDB_env* const env, +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + const char* const path, + const unsigned int flags = 0) { + const int rc = ::mdb_env_copy2(env, path, flags); +#else + const char* const path) { + const int rc = ::mdb_env_copy(env, path); +#endif + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_copy2", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga5040d0de1f14000fa01fc0b522ff1f86 + * @see http://symas.com/mdb/doc/group__mdb.html#ga470b0bcc64ac417de5de5930f20b1a28 + */ +static inline void +lmdb::env_copy_fd(MDB_env* const env, +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + const mdb_filehandle_t fd, + const unsigned int flags = 0) { + const int rc = ::mdb_env_copyfd2(env, fd, flags); +#else + const mdb_filehandle_t fd) { + const int rc = ::mdb_env_copyfd(env, fd); +#endif + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_copyfd2", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaf881dca452050efbd434cd16e4bae255 + */ +static inline void +lmdb::env_stat(MDB_env* const env, + MDB_stat* const stat) { + const int rc = ::mdb_env_stat(env, stat); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_stat", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga18769362c7e7d6cf91889a028a5c5947 + */ +static inline void +lmdb::env_info(MDB_env* const env, + MDB_envinfo* const stat) { + const int rc = ::mdb_env_info(env, stat); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_info", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga85e61f05aa68b520cc6c3b981dba5037 + */ +static inline void +lmdb::env_sync(MDB_env* const env, + const bool force = true) { + const int rc = ::mdb_env_sync(env, force); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_sync", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga4366c43ada8874588b6a62fbda2d1e95 + */ +static inline void +lmdb::env_close(MDB_env* const env) noexcept { + ::mdb_env_close(env); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga83f66cf02bfd42119451e9468dc58445 + */ +static inline void +lmdb::env_set_flags(MDB_env* const env, + const unsigned int flags, + const bool onoff = true) { + const int rc = ::mdb_env_set_flags(env, flags, onoff ? 1 : 0); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_flags", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga2733aefc6f50beb49dd0c6eb19b067d9 + */ +static inline void +lmdb::env_get_flags(MDB_env* const env, + unsigned int* const flags) { + const int rc = ::mdb_env_get_flags(env, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_flags", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac699fdd8c4f8013577cb933fb6a757fe + */ +static inline void +lmdb::env_get_path(MDB_env* const env, + const char** path) { + const int rc = ::mdb_env_get_path(env, path); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_path", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaf1570e7c0e5a5d860fef1032cec7d5f2 + */ +static inline void +lmdb::env_get_fd(MDB_env* const env, + mdb_filehandle_t* const fd) { + const int rc = ::mdb_env_get_fd(env, fd); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_fd", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5 + */ +static inline void +lmdb::env_set_mapsize(MDB_env* const env, + const std::size_t size) { + const int rc = ::mdb_env_set_mapsize(env, size); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_mapsize", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gae687966c24b790630be2a41573fe40e2 + */ +static inline void +lmdb::env_set_max_readers(MDB_env* const env, + const unsigned int count) { + const int rc = ::mdb_env_set_maxreaders(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_maxreaders", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga70e143cf11760d869f754c9c9956e6cc + */ +static inline void +lmdb::env_get_max_readers(MDB_env* const env, + unsigned int* const count) { + const int rc = ::mdb_env_get_maxreaders(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_maxreaders", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaa2fc2f1f37cb1115e733b62cab2fcdbc + */ +static inline void +lmdb::env_set_max_dbs(MDB_env* const env, + const MDB_dbi count) { + const int rc = ::mdb_env_set_maxdbs(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_maxdbs", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94 + */ +static inline unsigned int +lmdb::env_get_max_keysize(MDB_env* const env) { + const int rc = ::mdb_env_get_maxkeysize(env); +#ifdef LMDBXX_DEBUG + assert(rc >= 0); +#endif + return static_cast(rc); +} + +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) +/** + * @throws lmdb::error on failure + * @since 0.9.11 (2014/01/15) + * @see http://symas.com/mdb/doc/group__mdb.html#gaf2fe09eb9c96eeb915a76bf713eecc46 + */ +static inline void +lmdb::env_set_userctx(MDB_env* const env, + void* const ctx) { + const int rc = ::mdb_env_set_userctx(env, ctx); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_userctx", rc); + } +} +#endif + +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) +/** + * @since 0.9.11 (2014/01/15) + * @see http://symas.com/mdb/doc/group__mdb.html#ga45df6a4fb150cda2316b5ae224ba52f1 + */ +static inline void* +lmdb::env_get_userctx(MDB_env* const env) { + return ::mdb_env_get_userctx(env); +} +#endif + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Transactions */ + +namespace lmdb { + static inline void txn_begin( + MDB_env* env, MDB_txn* parent, unsigned int flags, MDB_txn** txn); + static inline MDB_env* txn_env(MDB_txn* txn) noexcept; +#ifdef LMDBXX_TXN_ID + static inline std::size_t txn_id(MDB_txn* txn) noexcept; +#endif + static inline void txn_commit(MDB_txn* txn); + static inline void txn_abort(MDB_txn* txn) noexcept; + static inline void txn_reset(MDB_txn* txn) noexcept; + static inline void txn_renew(MDB_txn* txn); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gad7ea55da06b77513609efebd44b26920 + */ +static inline void +lmdb::txn_begin(MDB_env* const env, + MDB_txn* const parent, + const unsigned int flags, + MDB_txn** txn) { + const int rc = ::mdb_txn_begin(env, parent, flags, txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_begin", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gaeb17735b8aaa2938a78a45cab85c06a0 + */ +static inline MDB_env* +lmdb::txn_env(MDB_txn* const txn) noexcept { + return ::mdb_txn_env(txn); +} + +#ifdef LMDBXX_TXN_ID +/** + * @note Only available in HEAD, not yet in any 0.9.x release (as of 0.9.16). + */ +static inline std::size_t +lmdb::txn_id(MDB_txn* const txn) noexcept { + return ::mdb_txn_id(txn); +} +#endif + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga846fbd6f46105617ac9f4d76476f6597 + */ +static inline void +lmdb::txn_commit(MDB_txn* const txn) { + const int rc = ::mdb_txn_commit(txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_commit", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga73a5938ae4c3239ee11efa07eb22b882 + */ +static inline void +lmdb::txn_abort(MDB_txn* const txn) noexcept { + ::mdb_txn_abort(txn); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga02b06706f8a66249769503c4e88c56cd + */ +static inline void +lmdb::txn_reset(MDB_txn* const txn) noexcept { + ::mdb_txn_reset(txn); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga6c6f917959517ede1c504cf7c720ce6d + */ +static inline void +lmdb::txn_renew(MDB_txn* const txn) { + const int rc = ::mdb_txn_renew(txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_renew", rc); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Databases */ + +namespace lmdb { + static inline void dbi_open( + MDB_txn* txn, const char* name, unsigned int flags, MDB_dbi* dbi); + static inline void dbi_stat(MDB_txn* txn, MDB_dbi dbi, MDB_stat* stat); + static inline void dbi_flags(MDB_txn* txn, MDB_dbi dbi, unsigned int* flags); + static inline void dbi_close(MDB_env* env, MDB_dbi dbi) noexcept; + static inline void dbi_drop(MDB_txn* txn, MDB_dbi dbi, bool del); + static inline void dbi_set_compare(MDB_txn* txn, MDB_dbi dbi, MDB_cmp_func* cmp); + static inline void dbi_set_dupsort(MDB_txn* txn, MDB_dbi dbi, MDB_cmp_func* cmp); + static inline void dbi_set_relfunc(MDB_txn* txn, MDB_dbi dbi, MDB_rel_func* rel); + static inline void dbi_set_relctx(MDB_txn* txn, MDB_dbi dbi, void* ctx); + static inline bool dbi_get(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, MDB_val* data); + static inline bool dbi_put(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, MDB_val* data, unsigned int flags); + static inline bool dbi_del(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, const MDB_val* data); + // TODO: mdb_cmp() + // TODO: mdb_dcmp() +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac08cad5b096925642ca359a6d6f0562a + */ +static inline void +lmdb::dbi_open(MDB_txn* const txn, + const char* const name, + const unsigned int flags, + MDB_dbi* const dbi) { + const int rc = ::mdb_dbi_open(txn, name, flags, dbi); + if (rc != MDB_SUCCESS) { + error::raise("mdb_dbi_open", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gae6c1069febe94299769dbdd032fadef6 + */ +static inline void +lmdb::dbi_stat(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_stat* const result) { + const int rc = ::mdb_stat(txn, dbi, result); + if (rc != MDB_SUCCESS) { + error::raise("mdb_stat", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga95ba4cb721035478a8705e57b91ae4d4 + */ +static inline void +lmdb::dbi_flags(MDB_txn* const txn, + const MDB_dbi dbi, + unsigned int* const flags) { + const int rc = ::mdb_dbi_flags(txn, dbi, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_dbi_flags", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga52dd98d0c542378370cd6b712ff961b5 + */ +static inline void +lmdb::dbi_close(MDB_env* const env, + const MDB_dbi dbi) noexcept { + ::mdb_dbi_close(env, dbi); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gab966fab3840fc54a6571dfb32b00f2db + */ +static inline void +lmdb::dbi_drop(MDB_txn* const txn, + const MDB_dbi dbi, + const bool del = false) { + const int rc = ::mdb_drop(txn, dbi, del ? 1 : 0); + if (rc != MDB_SUCCESS) { + error::raise("mdb_drop", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga68e47ffcf72eceec553c72b1784ee0fe + */ +static inline void +lmdb::dbi_set_compare(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cmp_func* const cmp = nullptr) { + const int rc = ::mdb_set_compare(txn, dbi, cmp); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_compare", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gacef4ec3dab0bbd9bc978b73c19c879ae + */ +static inline void +lmdb::dbi_set_dupsort(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cmp_func* const cmp = nullptr) { + const int rc = ::mdb_set_dupsort(txn, dbi, cmp); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_dupsort", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga697d82c7afe79f142207ad5adcdebfeb + */ +static inline void +lmdb::dbi_set_relfunc(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_rel_func* const rel) { + const int rc = ::mdb_set_relfunc(txn, dbi, rel); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_relfunc", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga7c34246308cee01724a1839a8f5cc594 + */ +static inline void +lmdb::dbi_set_relctx(MDB_txn* const txn, + const MDB_dbi dbi, + void* const ctx) { + const int rc = ::mdb_set_relctx(txn, dbi, ctx); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_relctx", rc); + } +} + +/** + * @retval true if the key/value pair was retrieved + * @retval false if the key wasn't found + * @see http://symas.com/mdb/doc/group__mdb.html#ga8bf10cd91d3f3a83a34d04ce6b07992d + */ +static inline bool +lmdb::dbi_get(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + MDB_val* const data) { + const int rc = ::mdb_get(txn, dbi, const_cast(key), data); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_get", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @retval true if the key/value pair was inserted + * @retval false if the key already existed + * @see http://symas.com/mdb/doc/group__mdb.html#ga4fa8573d9236d54687c61827ebf8cac0 + */ +static inline bool +lmdb::dbi_put(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + MDB_val* const data, + const unsigned int flags = 0) { + const int rc = ::mdb_put(txn, dbi, const_cast(key), data, flags); + if (rc != MDB_SUCCESS && rc != MDB_KEYEXIST) { + error::raise("mdb_put", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @retval true if the key/value pair was removed + * @retval false if the key wasn't found + * @see http://symas.com/mdb/doc/group__mdb.html#gab8182f9360ea69ac0afd4a4eaab1ddb0 + */ +static inline bool +lmdb::dbi_del(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + const MDB_val* const data = nullptr) { + const int rc = ::mdb_del(txn, dbi, const_cast(key), const_cast(data)); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_del", rc); + } + return (rc == MDB_SUCCESS); +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Cursors */ + +namespace lmdb { + static inline void cursor_open(MDB_txn* txn, MDB_dbi dbi, MDB_cursor** cursor); + static inline void cursor_close(MDB_cursor* cursor) noexcept; + static inline void cursor_renew(MDB_txn* txn, MDB_cursor* cursor); + static inline MDB_txn* cursor_txn(MDB_cursor* cursor) noexcept; + static inline MDB_dbi cursor_dbi(MDB_cursor* cursor) noexcept; + static inline bool cursor_get(MDB_cursor* cursor, MDB_val* key, MDB_val* data, MDB_cursor_op op); + static inline void cursor_put(MDB_cursor* cursor, MDB_val* key, MDB_val* data, unsigned int flags); + static inline void cursor_del(MDB_cursor* cursor, unsigned int flags); + static inline void cursor_count(MDB_cursor* cursor, std::size_t& count); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga9ff5d7bd42557fd5ee235dc1d62613aa + */ +static inline void +lmdb::cursor_open(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cursor** const cursor) { + const int rc = ::mdb_cursor_open(txn, dbi, cursor); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_open", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gad685f5d73c052715c7bd859cc4c05188 + */ +static inline void +lmdb::cursor_close(MDB_cursor* const cursor) noexcept { + ::mdb_cursor_close(cursor); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac8b57befb68793070c85ea813df481af + */ +static inline void +lmdb::cursor_renew(MDB_txn* const txn, + MDB_cursor* const cursor) { + const int rc = ::mdb_cursor_renew(txn, cursor); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_renew", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga7bf0d458f7f36b5232fcb368ebda79e0 + */ +static inline MDB_txn* +lmdb::cursor_txn(MDB_cursor* const cursor) noexcept { + return ::mdb_cursor_txn(cursor); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga2f7092cf70ee816fb3d2c3267a732372 + */ +static inline MDB_dbi +lmdb::cursor_dbi(MDB_cursor* const cursor) noexcept { + return ::mdb_cursor_dbi(cursor); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga48df35fb102536b32dfbb801a47b4cb0 + */ +static inline bool +lmdb::cursor_get(MDB_cursor* const cursor, + MDB_val* const key, + MDB_val* const data, + const MDB_cursor_op op) { + const int rc = ::mdb_cursor_get(cursor, key, data, op); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_cursor_get", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga1f83ccb40011837ff37cc32be01ad91e + */ +static inline void +lmdb::cursor_put(MDB_cursor* const cursor, + MDB_val* const key, + MDB_val* const data, + const unsigned int flags = 0) { + const int rc = ::mdb_cursor_put(cursor, key, data, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_put", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga26a52d3efcfd72e5bf6bd6960bf75f95 + */ +static inline void +lmdb::cursor_del(MDB_cursor* const cursor, + const unsigned int flags = 0) { + const int rc = ::mdb_cursor_del(cursor, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_del", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga4041fd1e1862c6b7d5f10590b86ffbe2 + */ +static inline void +lmdb::cursor_count(MDB_cursor* const cursor, + std::size_t& count) { + const int rc = ::mdb_cursor_count(cursor, &count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_count", rc); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Values */ + +namespace lmdb { + class val; +} + +/** + * Wrapper class for `MDB_val` structures. + * + * @note Instances of this class are movable and copyable both. + * @see http://symas.com/mdb/doc/group__mdb.html#structMDB__val + */ +class lmdb::val { +protected: + MDB_val _val; + +public: + /** + * Default constructor. + */ + val() noexcept = default; + + /** + * Constructor. + */ + val(const std::string& data) noexcept + : val{data.data(), data.size()} {} + + /** + * Constructor. + */ + val(const char* const data) noexcept + : val{data, std::strlen(data)} {} + + /** + * Constructor. + */ + val(const void* const data, + const std::size_t size) noexcept + : _val{size, const_cast(data)} {} + + /** + * Move constructor. + */ + val(val&& other) noexcept = default; + + /** + * Move assignment operator. + */ + val& operator=(val&& other) noexcept = default; + + /** + * Destructor. + */ + ~val() noexcept = default; + + /** + * Returns an `MDB_val*` pointer. + */ + operator MDB_val*() noexcept { + return &_val; + } + + /** + * Returns an `MDB_val*` pointer. + */ + operator const MDB_val*() const noexcept { + return &_val; + } + + /** + * Determines whether this value is empty. + */ + bool empty() const noexcept { + return size() == 0; + } + + /** + * Returns the size of the data. + */ + std::size_t size() const noexcept { + return _val.mv_size; + } + + /** + * Returns a pointer to the data. + */ + template + T* data() noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + template + const T* data() const noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + char* data() noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + const char* data() const noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Assigns the value. + */ + template + val& assign(const T* const data, + const std::size_t size) noexcept { + _val.mv_size = size; + _val.mv_data = const_cast(reinterpret_cast(data)); + return *this; + } + + /** + * Assigns the value. + */ + val& assign(const char* const data) noexcept { + return assign(data, std::strlen(data)); + } + + /** + * Assigns the value. + */ + val& assign(const std::string& data) noexcept { + return assign(data.data(), data.size()); + } +}; + +#if !(defined(__COVERITY__) || defined(_MSC_VER)) +static_assert(std::is_pod::value, "lmdb::val must be a POD type"); +static_assert(sizeof(lmdb::val) == sizeof(MDB_val), "sizeof(lmdb::val) != sizeof(MDB_val)"); +#endif + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Environment */ + +namespace lmdb { + class env; +} + +/** + * Resource class for `MDB_env*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__env + */ +class lmdb::env { +protected: + MDB_env* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + static constexpr mode default_mode = 0644; /* -rw-r--r-- */ + + /** + * Creates a new LMDB environment. + * + * @param flags + * @throws lmdb::error on failure + */ + static env create(const unsigned int flags = default_flags) { + MDB_env* handle{nullptr}; + lmdb::env_create(&handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + if (flags) { + try { + lmdb::env_set_flags(handle, flags); + } + catch (const lmdb::error&) { + lmdb::env_close(handle); + throw; + } + } + return env{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_env*` handle + */ + env(MDB_env* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + env(env&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + env& operator=(env&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~env() noexcept { + try { close(); } catch (...) {} + } + + /** + * Returns the underlying `MDB_env*` handle. + */ + operator MDB_env*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_env*` handle. + */ + MDB_env* handle() const noexcept { + return _handle; + } + + /** + * Flushes data buffers to disk. + * + * @param force + * @throws lmdb::error on failure + */ + void sync(const bool force = true) { + lmdb::env_sync(handle(), force); + } + + /** + * Closes this environment, releasing the memory map. + * + * @note this method is idempotent + * @post `handle() == nullptr` + */ + void close() noexcept { + if (handle()) { + lmdb::env_close(handle()); + _handle = nullptr; + } + } + + /** + * Opens this environment. + * + * @param path + * @param flags + * @param mode + * @throws lmdb::error on failure + */ + env& open(const char* const path, + const unsigned int flags = default_flags, + const mode mode = default_mode) { + lmdb::env_open(handle(), path, flags, mode); + return *this; + } + + /** + * @param flags + * @param onoff + * @throws lmdb::error on failure + */ + env& set_flags(const unsigned int flags, + const bool onoff = true) { + lmdb::env_set_flags(handle(), flags, onoff); + return *this; + } + + /** + * @param size + * @throws lmdb::error on failure + */ + env& set_mapsize(const std::size_t size) { + lmdb::env_set_mapsize(handle(), size); + return *this; + } + + /** + * @param count + * @throws lmdb::error on failure + */ + env& set_max_readers(const unsigned int count) { + lmdb::env_set_max_readers(handle(), count); + return *this; + } + + /** + * @param count + * @throws lmdb::error on failure + */ + env& set_max_dbs(const MDB_dbi count) { + lmdb::env_set_max_dbs(handle(), count); + return *this; + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Transactions */ + +namespace lmdb { + class txn; +} + +/** + * Resource class for `MDB_txn*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__txn + */ +class lmdb::txn { +protected: + MDB_txn* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + + /** + * Creates a new LMDB transaction. + * + * @param env the environment handle + * @param parent + * @param flags + * @throws lmdb::error on failure + */ + static txn begin(MDB_env* const env, + MDB_txn* const parent = nullptr, + const unsigned int flags = default_flags) { + MDB_txn* handle{nullptr}; + lmdb::txn_begin(env, parent, flags, &handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + return txn{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_txn*` handle + */ + txn(MDB_txn* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + txn(txn&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + txn& operator=(txn&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~txn() noexcept { + if (_handle) { + try { abort(); } catch (...) {} + _handle = nullptr; + } + } + + /** + * Returns the underlying `MDB_txn*` handle. + */ + operator MDB_txn*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_txn*` handle. + */ + MDB_txn* handle() const noexcept { + return _handle; + } + + /** + * Returns the transaction's `MDB_env*` handle. + */ + MDB_env* env() const noexcept { + return lmdb::txn_env(handle()); + } + + /** + * Commits this transaction. + * + * @throws lmdb::error on failure + * @post `handle() == nullptr` + */ + void commit() { + lmdb::txn_commit(_handle); + _handle = nullptr; + } + + /** + * Aborts this transaction. + * + * @post `handle() == nullptr` + */ + void abort() noexcept { + lmdb::txn_abort(_handle); + _handle = nullptr; + } + + /** + * Resets this read-only transaction. + */ + void reset() noexcept { + lmdb::txn_reset(_handle); + } + + /** + * Renews this read-only transaction. + * + * @throws lmdb::error on failure + */ + void renew() { + lmdb::txn_renew(_handle); + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Databases */ + +namespace lmdb { + class dbi; +} + +/** + * Resource class for `MDB_dbi` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__mdb.html#gadbe68a06c448dfb62da16443d251a78b + */ +class lmdb::dbi { +protected: + MDB_dbi _handle{0}; + +public: + static constexpr unsigned int default_flags = 0; + static constexpr unsigned int default_put_flags = 0; + + /** + * Opens a database handle. + * + * @param txn the transaction handle + * @param name + * @param flags + * @throws lmdb::error on failure + */ + static dbi + open(MDB_txn* const txn, + const char* const name = nullptr, + const unsigned int flags = default_flags) { + MDB_dbi handle{}; + lmdb::dbi_open(txn, name, flags, &handle); + return dbi{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_dbi` handle + */ + dbi(const MDB_dbi handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + dbi(dbi&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + dbi& operator=(dbi&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~dbi() noexcept { + if (_handle) { + /* No need to call close() here. */ + } + } + + /** + * Returns the underlying `MDB_dbi` handle. + */ + operator MDB_dbi() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_dbi` handle. + */ + MDB_dbi handle() const noexcept { + return _handle; + } + + /** + * Returns statistics for this database. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + MDB_stat stat(MDB_txn* const txn) const { + MDB_stat result; + lmdb::dbi_stat(txn, handle(), &result); + return result; + } + + /** + * Retrieves the flags for this database handle. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + unsigned int flags(MDB_txn* const txn) const { + unsigned int result{}; + lmdb::dbi_flags(txn, handle(), &result); + return result; + } + + /** + * Returns the number of records in this database. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + std::size_t size(MDB_txn* const txn) const { + return stat(txn).ms_entries; + } + + /** + * @param txn a transaction handle + * @param del + * @throws lmdb::error on failure + */ + void drop(MDB_txn* const txn, + const bool del = false) { + lmdb::dbi_drop(txn, handle(), del); + } + + /** + * Sets a custom key comparison function for this database. + * + * @param txn a transaction handle + * @param cmp the comparison function + * @throws lmdb::error on failure + */ + dbi& set_compare(MDB_txn* const txn, + MDB_cmp_func* const cmp = nullptr) { + lmdb::dbi_set_compare(txn, handle(), cmp); + return *this; + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @param data + * @throws lmdb::error on failure + */ + bool get(MDB_txn* const txn, + const val& key, + val& data) { + return lmdb::dbi_get(txn, handle(), key, data); + } + + /** + * Retrieves a key from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const K& key) const { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + return lmdb::dbi_get(txn, handle(), k, v); + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @param val + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const K& key, + V& val) const { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + const bool result = lmdb::dbi_get(txn, handle(), k, v); + if (result) { + val = *v.data(); + } + return result; + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const char* const key, + V& val) const { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{}; + const bool result = lmdb::dbi_get(txn, handle(), k, v); + if (result) { + val = *v.data(); + } + return result; + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key + * @param data + * @param flags + * @throws lmdb::error on failure + */ + bool put(MDB_txn* const txn, + const val& key, + val& data, + const unsigned int flags = default_put_flags) { + return lmdb::dbi_put(txn, handle(), key, data, flags); + } + + /** + * Stores a key into this database. + * + * @param txn a transaction handle + * @param key + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const K& key, + const unsigned int flags = default_put_flags) { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key + * @param val + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const K& key, + const V& val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{&val, sizeof(V)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const char* const key, + const V& val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{&val, sizeof(V)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val a NUL-terminated string key + * @param flags + * @throws lmdb::error on failure + */ + bool put(MDB_txn* const txn, + const char* const key, + const char* const val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{val, std::strlen(val)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Removes a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + bool del(MDB_txn* const txn, + const val& key) { + return lmdb::dbi_del(txn, handle(), key); + } + + /** + * Removes a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + template + bool del(MDB_txn* const txn, + const K& key) { + const lmdb::val k{&key, sizeof(K)}; + return lmdb::dbi_del(txn, handle(), k); + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Cursors */ + +namespace lmdb { + class cursor; +} + +/** + * Resource class for `MDB_cursor*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__cursor + */ +class lmdb::cursor { +protected: + MDB_cursor* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + + /** + * Creates an LMDB cursor. + * + * @param txn the transaction handle + * @param dbi the database handle + * @throws lmdb::error on failure + */ + static cursor + open(MDB_txn* const txn, + const MDB_dbi dbi) { + MDB_cursor* handle{}; + lmdb::cursor_open(txn, dbi, &handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + return cursor{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_cursor*` handle + */ + cursor(MDB_cursor* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + cursor(cursor&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + cursor& operator=(cursor&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~cursor() noexcept { + try { close(); } catch (...) {} + } + + /** + * Returns the underlying `MDB_cursor*` handle. + */ + operator MDB_cursor*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_cursor*` handle. + */ + MDB_cursor* handle() const noexcept { + return _handle; + } + + /** + * Closes this cursor. + * + * @note this method is idempotent + * @post `handle() == nullptr` + */ + void close() noexcept { + if (_handle) { + lmdb::cursor_close(_handle); + _handle = nullptr; + } + } + + /** + * Renews this cursor. + * + * @param txn the transaction scope + * @throws lmdb::error on failure + */ + void renew(MDB_txn* const txn) { + lmdb::cursor_renew(txn, handle()); + } + + /** + * Returns the cursor's transaction handle. + */ + MDB_txn* txn() const noexcept { + return lmdb::cursor_txn(handle()); + } + + /** + * Returns the cursor's database handle. + */ + MDB_dbi dbi() const noexcept { + return lmdb::cursor_dbi(handle()); + } + + /** + * Retrieves a key from the database. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + bool get(MDB_val* const key, + const MDB_cursor_op op) { + return get(key, nullptr, op); + } + + /** + * Retrieves a key from the database. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + bool get(lmdb::val& key, + const MDB_cursor_op op) { + return get(key, nullptr, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val (may be `nullptr`) + * @param op + * @throws lmdb::error on failure + */ + bool get(MDB_val* const key, + MDB_val* const val, + const MDB_cursor_op op) { + return lmdb::cursor_get(handle(), key, val, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val + * @param op + * @throws lmdb::error on failure + */ + bool get(lmdb::val& key, + lmdb::val& val, + const MDB_cursor_op op) { + return lmdb::cursor_get(handle(), key, val, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val + * @param op + * @throws lmdb::error on failure + */ + bool get(std::string& key, + std::string& val, + const MDB_cursor_op op) { + lmdb::val k{}, v{}; + const bool found = get(k, v, op); + if (found) { + key.assign(k.data(), k.size()); + val.assign(v.data(), v.size()); + } + return found; + } + + /** + * Positions this cursor at the given key. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + template + bool find(const K& key, + const MDB_cursor_op op = MDB_SET) { + lmdb::val k{&key, sizeof(K)}; + return get(k, nullptr, op); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +#endif /* LMDBXX_H */ diff --git a/cpp/include/prettyprint.hpp b/cpp/include/prettyprint.hpp new file mode 100644 index 0000000..ebba086 --- /dev/null +++ b/cpp/include/prettyprint.hpp @@ -0,0 +1,446 @@ +// https://github.com/louisdx/cxx-prettyprint/blob/master/prettyprint.hpp 27cad81cabff867ed3f72fdf7a195d2c2a90552f +// Copyright Louis Delacroix 2010 - 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// +// A pretty printing library for C++ +// +// Usage: +// Include this header, and operator<< will "just work". + +#ifndef H_PRETTY_PRINT +#define H_PRETTY_PRINT + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pretty_print +{ + namespace detail + { + // SFINAE type trait to detect whether T::const_iterator exists. + + struct sfinae_base + { + using yes = char; + using no = yes[2]; + }; + + template + struct has_const_iterator : private sfinae_base + { + private: + template static yes & test(typename C::const_iterator*); + template static no & test(...); + public: + static const bool value = sizeof(test(nullptr)) == sizeof(yes); + using type = T; + }; + + template + struct has_begin_end : private sfinae_base + { + private: + template + static yes & f(typename std::enable_if< + std::is_same(&C::begin)), + typename C::const_iterator(C::*)() const>::value>::type *); + + template static no & f(...); + + template + static yes & g(typename std::enable_if< + std::is_same(&C::end)), + typename C::const_iterator(C::*)() const>::value, void>::type*); + + template static no & g(...); + + public: + static bool const beg_value = sizeof(f(nullptr)) == sizeof(yes); + static bool const end_value = sizeof(g(nullptr)) == sizeof(yes); + }; + + } // namespace detail + + + // Holds the delimiter values for a specific character type + + template + struct delimiters_values + { + using char_type = TChar; + const char_type * prefix; + const char_type * delimiter; + const char_type * postfix; + }; + + + // Defines the delimiter values for a specific container and character type + + template + struct delimiters + { + using type = delimiters_values; + static const type values; + }; + + + // Functor to print containers. You can use this directly if you want + // to specificy a non-default delimiters type. The printing logic can + // be customized by specializing the nested template. + + template , + typename TDelimiters = delimiters> + struct print_container_helper + { + using delimiters_type = TDelimiters; + using ostream_type = std::basic_ostream; + + template + struct printer + { + static void print_body(const U & c, ostream_type & stream) + { + using std::begin; + using std::end; + + auto it = begin(c); + const auto the_end = end(c); + + if (it != the_end) + { + for ( ; ; ) + { + stream << *it; + + if (++it == the_end) break; + + if (delimiters_type::values.delimiter != NULL) + stream << delimiters_type::values.delimiter; + } + } + } + }; + + print_container_helper(const T & container) + : container_(container) + { } + + inline void operator()(ostream_type & stream) const + { + if (delimiters_type::values.prefix != NULL) + stream << delimiters_type::values.prefix; + + printer::print_body(container_, stream); + + if (delimiters_type::values.postfix != NULL) + stream << delimiters_type::values.postfix; + } + + private: + const T & container_; + }; + + // Specialization for pairs + + template + template + struct print_container_helper::printer> + { + using ostream_type = typename print_container_helper::ostream_type; + + static void print_body(const std::pair & c, ostream_type & stream) + { + stream << c.first; + if (print_container_helper::delimiters_type::values.delimiter != NULL) + stream << print_container_helper::delimiters_type::values.delimiter; + stream << c.second; + } + }; + + // Specialization for tuples + + template + template + struct print_container_helper::printer> + { + using ostream_type = typename print_container_helper::ostream_type; + using element_type = std::tuple; + + template struct Int { }; + + static void print_body(const element_type & c, ostream_type & stream) + { + tuple_print(c, stream, Int<0>()); + } + + static void tuple_print(const element_type &, ostream_type &, Int) + { + } + + static void tuple_print(const element_type & c, ostream_type & stream, + typename std::conditional, std::nullptr_t>::type) + { + stream << std::get<0>(c); + tuple_print(c, stream, Int<1>()); + } + + template + static void tuple_print(const element_type & c, ostream_type & stream, Int) + { + if (print_container_helper::delimiters_type::values.delimiter != NULL) + stream << print_container_helper::delimiters_type::values.delimiter; + + stream << std::get(c); + + tuple_print(c, stream, Int()); + } + }; + + // Prints a print_container_helper to the specified stream. + + template + inline std::basic_ostream & operator<<( + std::basic_ostream & stream, + const print_container_helper & helper) + { + helper(stream); + return stream; + } + + + // Basic is_container template; specialize to derive from std::true_type for all desired container types + + template + struct is_container : public std::integral_constant::value && + detail::has_begin_end::beg_value && + detail::has_begin_end::end_value> { }; + + template + struct is_container : std::true_type { }; + + template + struct is_container : std::false_type { }; + + template + struct is_container> : std::true_type { }; + + template + struct is_container> : std::true_type { }; + + template + struct is_container> : std::true_type { }; + + + // Default delimiters + + template struct delimiters { static const delimiters_values values; }; + template const delimiters_values delimiters::values = { "[", ", ", "]" }; + template struct delimiters { static const delimiters_values values; }; + template const delimiters_values delimiters::values = { L"[", L", ", L"]" }; + + + // Delimiters for (multi)set and unordered_(multi)set + + template + struct delimiters< ::std::set, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::set, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::set, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::set, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::multiset, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::multiset, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::multiset, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::multiset, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::unordered_set, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_set, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::unordered_set, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_set, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::unordered_multiset, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_multiset, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::unordered_multiset, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_multiset, wchar_t>::values = { L"{", L", ", L"}" }; + + + // Delimiters for pair and tuple + + template struct delimiters, char> { static const delimiters_values values; }; + template const delimiters_values delimiters, char>::values = { "(", ", ", ")" }; + template struct delimiters< ::std::pair, wchar_t> { static const delimiters_values values; }; + template const delimiters_values delimiters< ::std::pair, wchar_t>::values = { L"(", L", ", L")" }; + + template struct delimiters, char> { static const delimiters_values values; }; + template const delimiters_values delimiters, char>::values = { "(", ", ", ")" }; + template struct delimiters< ::std::tuple, wchar_t> { static const delimiters_values values; }; + template const delimiters_values delimiters< ::std::tuple, wchar_t>::values = { L"(", L", ", L")" }; + + + // Type-erasing helper class for easy use of custom delimiters. + // Requires TCharTraits = std::char_traits and TChar = char or wchar_t, and MyDelims needs to be defined for TChar. + // Usage: "cout << pretty_print::custom_delims(x)". + + struct custom_delims_base + { + virtual ~custom_delims_base() { } + virtual std::ostream & stream(::std::ostream &) = 0; + virtual std::wostream & stream(::std::wostream &) = 0; + }; + + template + struct custom_delims_wrapper : custom_delims_base + { + custom_delims_wrapper(const T & t_) : t(t_) { } + + std::ostream & stream(std::ostream & s) + { + return s << print_container_helper, Delims>(t); + } + + std::wostream & stream(std::wostream & s) + { + return s << print_container_helper, Delims>(t); + } + + private: + const T & t; + }; + + template + struct custom_delims + { + template + custom_delims(const Container & c) : base(new custom_delims_wrapper(c)) { } + + std::unique_ptr base; + }; + + template + inline std::basic_ostream & operator<<(std::basic_ostream & s, const custom_delims & p) + { + return p.base->stream(s); + } + + + // A wrapper for a C-style array given as pointer-plus-size. + // Usage: std::cout << pretty_print_array(arr, n) << std::endl; + + template + struct array_wrapper_n + { + typedef const T * const_iterator; + typedef T value_type; + + array_wrapper_n(const T * const a, size_t n) : _array(a), _n(n) { } + inline const_iterator begin() const { return _array; } + inline const_iterator end() const { return _array + _n; } + + private: + const T * const _array; + size_t _n; + }; + + + // A wrapper for hash-table based containers that offer local iterators to each bucket. + // Usage: std::cout << bucket_print(m, 4) << std::endl; (Prints bucket 5 of container m.) + + template + struct bucket_print_wrapper + { + typedef typename T::const_local_iterator const_iterator; + typedef typename T::size_type size_type; + + const_iterator begin() const + { + return m_map.cbegin(n); + } + + const_iterator end() const + { + return m_map.cend(n); + } + + bucket_print_wrapper(const T & m, size_type bucket) : m_map(m), n(bucket) { } + + private: + const T & m_map; + const size_type n; + }; + +} // namespace pretty_print + + +// Global accessor functions for the convenience wrappers + +template +inline pretty_print::array_wrapper_n pretty_print_array(const T * const a, size_t n) +{ + return pretty_print::array_wrapper_n(a, n); +} + +template pretty_print::bucket_print_wrapper +bucket_print(const T & m, typename T::size_type n) +{ + return pretty_print::bucket_print_wrapper(m, n); +} + + +// Main magic entry point: An overload snuck into namespace std. +// Can we do better? + +namespace std +{ + // Prints a container to the stream using default delimiters + + template + inline typename enable_if< ::pretty_print::is_container::value, + basic_ostream &>::type + operator<<(basic_ostream & stream, const T & container) + { + return stream << ::pretty_print::print_container_helper(container); + } +} + + + +#endif // H_PRETTY_PRINT diff --git a/cpp/include/scope_exit.hpp b/cpp/include/scope_exit.hpp new file mode 100644 index 0000000..f7cab9c --- /dev/null +++ b/cpp/include/scope_exit.hpp @@ -0,0 +1,39 @@ +// https://gist.github.com/socantre/b45b3e23e6f1f4715f08 +#include + +#pragma once + +namespace socantre { + +template struct scope_exit { + // construction + explicit scope_exit(EF &&f) noexcept(true) + : exit_function(std::move(f)), execute_on_destruction{true} {} + // move + scope_exit(scope_exit &&rhs) noexcept(true) + : exit_function(std::move(rhs.exit_function)), + execute_on_destruction{rhs.execute_on_destruction} { + rhs.release(); + } + // release + ~scope_exit() noexcept(noexcept(this->exit_function())) { + if (execute_on_destruction) + this->exit_function(); + } + void release() noexcept(true) { this->execute_on_destruction = false; } + +private: + scope_exit(scope_exit const &) = delete; + void operator=(scope_exit const &) = delete; + scope_exit &operator=(scope_exit &&) = delete; + EF exit_function; + bool execute_on_destruction; // exposition only +}; + +template +auto make_scope_exit(EF &&exit_function) noexcept(true) -> decltype( + scope_exit>(std::forward(exit_function))) { + return scope_exit>( + std::forward(exit_function)); +} +} // namespace socantre diff --git a/cpp/include/tinyformat.h b/cpp/include/tinyformat.h new file mode 100644 index 0000000..1194d75 --- /dev/null +++ b/cpp/include/tinyformat.h @@ -0,0 +1,1003 @@ +// tinyformat.h +// Copyright (C) 2011, Chris Foster [chris42f (at) gmail (d0t) com] +// +// Boost Software License - Version 1.0 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//------------------------------------------------------------------------------ +// Tinyformat: A minimal type safe printf replacement +// +// tinyformat.h is a type safe printf replacement library in a single C++ +// header file. Design goals include: +// +// * Type safety and extensibility for user defined types. +// * C99 printf() compatibility, to the extent possible using std::ostream +// * Simplicity and minimalism. A single header file to include and distribute +// with your projects. +// * Augment rather than replace the standard stream formatting mechanism +// * C++98 support, with optional C++11 niceties +// +// +// Main interface example usage +// ---------------------------- +// +// To print a date to std::cout: +// +// std::string weekday = "Wednesday"; +// const char* month = "July"; +// size_t day = 27; +// long hour = 14; +// int min = 44; +// +// tfm::printf("%s, %s %d, %.2d:%.2d\n", weekday, month, day, hour, min); +// +// The strange types here emphasize the type safety of the interface; it is +// possible to print a std::string using the "%s" conversion, and a +// size_t using the "%d" conversion. A similar result could be achieved +// using either of the tfm::format() functions. One prints on a user provided +// stream: +// +// tfm::format(std::cerr, "%s, %s %d, %.2d:%.2d\n", +// weekday, month, day, hour, min); +// +// The other returns a std::string: +// +// std::string date = tfm::format("%s, %s %d, %.2d:%.2d\n", +// weekday, month, day, hour, min); +// std::cout << date; +// +// These are the three primary interface functions. +// +// +// User defined format functions +// ----------------------------- +// +// Simulating variadic templates in C++98 is pretty painful since it requires +// writing out the same function for each desired number of arguments. To make +// this bearable tinyformat comes with a set of macros which are used +// internally to generate the API, but which may also be used in user code. +// +// The three macros TINYFORMAT_ARGTYPES(n), TINYFORMAT_VARARGS(n) and +// TINYFORMAT_PASSARGS(n) will generate a list of n argument types, +// type/name pairs and argument names respectively when called with an integer +// n between 1 and 16. We can use these to define a macro which generates the +// desired user defined function with n arguments. To generate all 16 user +// defined function bodies, use the macro TINYFORMAT_FOREACH_ARGNUM. For an +// example, see the implementation of printf() at the end of the source file. +// +// Sometimes it's useful to be able to pass a list of format arguments through +// to a non-template function. The FormatList class is provided as a way to do +// this by storing the argument list in a type-opaque way. Continuing the +// example from above, we construct a FormatList using makeFormatList(): +// +// FormatListRef formatList = tfm::makeFormatList(weekday, month, day, hour, min); +// +// The format list can now be passed into any non-template function and used +// via a call to the vformat() function: +// +// tfm::vformat(std::cout, "%s, %s %d, %.2d:%.2d\n", formatList); +// +// +// Additional API information +// -------------------------- +// +// Error handling: Define TINYFORMAT_ERROR to customize the error handling for +// format strings which are unsupported or have the wrong number of format +// specifiers (calls assert() by default). +// +// User defined types: Uses operator<< for user defined types by default. +// Overload formatValue() for more control. + + +#ifndef TINYFORMAT_H_INCLUDED +#define TINYFORMAT_H_INCLUDED + +namespace tinyformat {} +//------------------------------------------------------------------------------ +// Config section. Customize to your liking! + +// Namespace alias to encourage brevity +namespace tfm = tinyformat; + +// Error handling; calls assert() by default. +// #define TINYFORMAT_ERROR(reasonString) your_error_handler(reasonString) + +// Define for C++11 variadic templates which make the code shorter & more +// general. If you don't define this, C++11 support is autodetected below. +// #define TINYFORMAT_USE_VARIADIC_TEMPLATES + + +//------------------------------------------------------------------------------ +// Implementation details. +#include +#include +#include +#include + +#ifndef TINYFORMAT_ERROR +# define TINYFORMAT_ERROR(reason) assert(0 && reason) +#endif + +#if !defined(TINYFORMAT_USE_VARIADIC_TEMPLATES) && !defined(TINYFORMAT_NO_VARIADIC_TEMPLATES) +# ifdef __GXX_EXPERIMENTAL_CXX0X__ +# define TINYFORMAT_USE_VARIADIC_TEMPLATES +# endif +#endif + +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20080201 +// std::showpos is broken on old libstdc++ as provided with OSX. See +// http://gcc.gnu.org/ml/libstdc++/2007-11/msg00075.html +# define TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND +#endif + +namespace tinyformat { + +//------------------------------------------------------------------------------ +namespace detail { + +// Test whether type T1 is convertible to type T2 +template +struct is_convertible +{ + private: + // two types of different size + struct fail { char dummy[2]; }; + struct succeed { char dummy; }; + // Try to convert a T1 to a T2 by plugging into tryConvert + static fail tryConvert(...); + static succeed tryConvert(const T2&); + static const T1& makeT1(); + public: +# ifdef _MSC_VER + // Disable spurious loss of precision warnings in tryConvert(makeT1()) +# pragma warning(push) +# pragma warning(disable:4244) +# pragma warning(disable:4267) +# endif + // Standard trick: the (...) version of tryConvert will be chosen from + // the overload set only if the version taking a T2 doesn't match. + // Then we compare the sizes of the return types to check which + // function matched. Very neat, in a disgusting kind of way :) + static const bool value = + sizeof(tryConvert(makeT1())) == sizeof(succeed); +# ifdef _MSC_VER +# pragma warning(pop) +# endif +}; + + +// Detect when a type is not a wchar_t string +template struct is_wchar { typedef int tinyformat_wchar_is_not_supported; }; +template<> struct is_wchar {}; +template<> struct is_wchar {}; +template struct is_wchar {}; +template struct is_wchar {}; + + +// Format the value by casting to type fmtT. This default implementation +// should never be called. +template::value> +struct formatValueAsType +{ + static void invoke(std::ostream& /*out*/, const T& /*value*/) { assert(0); } +}; +// Specialized version for types that can actually be converted to fmtT, as +// indicated by the "convertible" template parameter. +template +struct formatValueAsType +{ + static void invoke(std::ostream& out, const T& value) + { out << static_cast(value); } +}; + +#ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND +template::value> +struct formatZeroIntegerWorkaround +{ + static bool invoke(std::ostream& /**/, const T& /**/) { return false; } +}; +template +struct formatZeroIntegerWorkaround +{ + static bool invoke(std::ostream& out, const T& value) + { + if (static_cast(value) == 0 && out.flags() & std::ios::showpos) + { + out << "+0"; + return true; + } + return false; + } +}; +#endif // TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND + +// Convert an arbitrary type to integer. The version with convertible=false +// throws an error. +template::value> +struct convertToInt +{ + static int invoke(const T& /*value*/) + { + TINYFORMAT_ERROR("tinyformat: Cannot convert from argument type to " + "integer for use as variable width or precision"); + return 0; + } +}; +// Specialization for convertToInt when conversion is possible +template +struct convertToInt +{ + static int invoke(const T& value) { return static_cast(value); } +}; + +// Format at most ntrunc characters to the given stream. +template +inline void formatTruncated(std::ostream& out, const T& value, int ntrunc) +{ + std::ostringstream tmp; + tmp << value; + std::string result = tmp.str(); + out.write(result.c_str(), std::min(ntrunc, static_cast(result.size()))); +} +#define TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(type) \ +inline void formatTruncated(std::ostream& out, type* value, int ntrunc) \ +{ \ + std::streamsize len = 0; \ + while(len < ntrunc && value[len] != 0) \ + ++len; \ + out.write(value, len); \ +} +// Overload for const char* and char*. Could overload for signed & unsigned +// char too, but these are technically unneeded for printf compatibility. +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(const char) +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(char) +#undef TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR + +} // namespace detail + + +//------------------------------------------------------------------------------ +// Variable formatting functions. May be overridden for user-defined types if +// desired. + + +/// Format a value into a stream, delegating to operator<< by default. +/// +/// Users may override this for their own types. When this function is called, +/// the stream flags will have been modified according to the format string. +/// The format specification is provided in the range [fmtBegin, fmtEnd). For +/// truncating conversions, ntrunc is set to the desired maximum number of +/// characters, for example "%.7s" calls formatValue with ntrunc = 7. +/// +/// By default, formatValue() uses the usual stream insertion operator +/// operator<< to format the type T, with special cases for the %c and %p +/// conversions. +template +inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, + const char* fmtEnd, int ntrunc, const T& value) +{ +#ifndef TINYFORMAT_ALLOW_WCHAR_STRINGS + // Since we don't support printing of wchar_t using "%ls", make it fail at + // compile time in preference to printing as a void* at runtime. + typedef typename detail::is_wchar::tinyformat_wchar_is_not_supported DummyType; + (void) DummyType(); // avoid unused type warning with gcc-4.8 +#endif + // The mess here is to support the %c and %p conversions: if these + // conversions are active we try to convert the type to a char or const + // void* respectively and format that instead of the value itself. For the + // %p conversion it's important to avoid dereferencing the pointer, which + // could otherwise lead to a crash when printing a dangling (const char*). + const bool canConvertToChar = detail::is_convertible::value; + const bool canConvertToVoidPtr = detail::is_convertible::value; + if(canConvertToChar && *(fmtEnd-1) == 'c') + detail::formatValueAsType::invoke(out, value); + else if(canConvertToVoidPtr && *(fmtEnd-1) == 'p') + detail::formatValueAsType::invoke(out, value); +#ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND + else if(detail::formatZeroIntegerWorkaround::invoke(out, value)) /**/; +#endif + else if(ntrunc >= 0) + { + // Take care not to overread C strings in truncating conversions like + // "%.4s" where at most 4 characters may be read. + detail::formatTruncated(out, value, ntrunc); + } + else + out << value; +} + + +// Overloaded version for char types to support printing as an integer +#define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \ +inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, \ + const char* fmtEnd, int /**/, charType value) \ +{ \ + switch(*(fmtEnd-1)) \ + { \ + case 'u': case 'd': case 'i': case 'o': case 'X': case 'x': \ + out << static_cast(value); break; \ + default: \ + out << value; break; \ + } \ +} +// per 3.9.1: char, signed char and unsigned char are all distinct types +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(char) +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(signed char) +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(unsigned char) +#undef TINYFORMAT_DEFINE_FORMATVALUE_CHAR + + +//------------------------------------------------------------------------------ +// Tools for emulating variadic templates in C++98. The basic idea here is +// stolen from the boost preprocessor metaprogramming library and cut down to +// be just general enough for what we need. + +#define TINYFORMAT_ARGTYPES(n) TINYFORMAT_ARGTYPES_ ## n +#define TINYFORMAT_VARARGS(n) TINYFORMAT_VARARGS_ ## n +#define TINYFORMAT_PASSARGS(n) TINYFORMAT_PASSARGS_ ## n +#define TINYFORMAT_PASSARGS_TAIL(n) TINYFORMAT_PASSARGS_TAIL_ ## n + +// To keep it as transparent as possible, the macros below have been generated +// using python via the excellent cog.py code generation script. This avoids +// the need for a bunch of complex (but more general) preprocessor tricks as +// used in boost.preprocessor. +// +// To rerun the code generation in place, use `cog.py -r tinyformat.h` +// (see http://nedbatchelder.com/code/cog). Alternatively you can just create +// extra versions by hand. + +/*[[[cog +maxParams = 16 + +def makeCommaSepLists(lineTemplate, elemTemplate, startInd=1): + for j in range(startInd,maxParams+1): + list = ', '.join([elemTemplate % {'i':i} for i in range(startInd,j+1)]) + cog.outl(lineTemplate % {'j':j, 'list':list}) + +makeCommaSepLists('#define TINYFORMAT_ARGTYPES_%(j)d %(list)s', + 'class T%(i)d') + +cog.outl() +makeCommaSepLists('#define TINYFORMAT_VARARGS_%(j)d %(list)s', + 'const T%(i)d& v%(i)d') + +cog.outl() +makeCommaSepLists('#define TINYFORMAT_PASSARGS_%(j)d %(list)s', 'v%(i)d') + +cog.outl() +cog.outl('#define TINYFORMAT_PASSARGS_TAIL_1') +makeCommaSepLists('#define TINYFORMAT_PASSARGS_TAIL_%(j)d , %(list)s', + 'v%(i)d', startInd = 2) + +cog.outl() +cog.outl('#define TINYFORMAT_FOREACH_ARGNUM(m) \\\n ' + + ' '.join(['m(%d)' % (j,) for j in range(1,maxParams+1)])) +]]]*/ +#define TINYFORMAT_ARGTYPES_1 class T1 +#define TINYFORMAT_ARGTYPES_2 class T1, class T2 +#define TINYFORMAT_ARGTYPES_3 class T1, class T2, class T3 +#define TINYFORMAT_ARGTYPES_4 class T1, class T2, class T3, class T4 +#define TINYFORMAT_ARGTYPES_5 class T1, class T2, class T3, class T4, class T5 +#define TINYFORMAT_ARGTYPES_6 class T1, class T2, class T3, class T4, class T5, class T6 +#define TINYFORMAT_ARGTYPES_7 class T1, class T2, class T3, class T4, class T5, class T6, class T7 +#define TINYFORMAT_ARGTYPES_8 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8 +#define TINYFORMAT_ARGTYPES_9 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9 +#define TINYFORMAT_ARGTYPES_10 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10 +#define TINYFORMAT_ARGTYPES_11 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11 +#define TINYFORMAT_ARGTYPES_12 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12 +#define TINYFORMAT_ARGTYPES_13 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13 +#define TINYFORMAT_ARGTYPES_14 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14 +#define TINYFORMAT_ARGTYPES_15 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14, class T15 +#define TINYFORMAT_ARGTYPES_16 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14, class T15, class T16 + +#define TINYFORMAT_VARARGS_1 const T1& v1 +#define TINYFORMAT_VARARGS_2 const T1& v1, const T2& v2 +#define TINYFORMAT_VARARGS_3 const T1& v1, const T2& v2, const T3& v3 +#define TINYFORMAT_VARARGS_4 const T1& v1, const T2& v2, const T3& v3, const T4& v4 +#define TINYFORMAT_VARARGS_5 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5 +#define TINYFORMAT_VARARGS_6 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6 +#define TINYFORMAT_VARARGS_7 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7 +#define TINYFORMAT_VARARGS_8 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8 +#define TINYFORMAT_VARARGS_9 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9 +#define TINYFORMAT_VARARGS_10 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10 +#define TINYFORMAT_VARARGS_11 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11 +#define TINYFORMAT_VARARGS_12 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12 +#define TINYFORMAT_VARARGS_13 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13 +#define TINYFORMAT_VARARGS_14 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14 +#define TINYFORMAT_VARARGS_15 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14, const T15& v15 +#define TINYFORMAT_VARARGS_16 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14, const T15& v15, const T16& v16 + +#define TINYFORMAT_PASSARGS_1 v1 +#define TINYFORMAT_PASSARGS_2 v1, v2 +#define TINYFORMAT_PASSARGS_3 v1, v2, v3 +#define TINYFORMAT_PASSARGS_4 v1, v2, v3, v4 +#define TINYFORMAT_PASSARGS_5 v1, v2, v3, v4, v5 +#define TINYFORMAT_PASSARGS_6 v1, v2, v3, v4, v5, v6 +#define TINYFORMAT_PASSARGS_7 v1, v2, v3, v4, v5, v6, v7 +#define TINYFORMAT_PASSARGS_8 v1, v2, v3, v4, v5, v6, v7, v8 +#define TINYFORMAT_PASSARGS_9 v1, v2, v3, v4, v5, v6, v7, v8, v9 +#define TINYFORMAT_PASSARGS_10 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 +#define TINYFORMAT_PASSARGS_11 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11 +#define TINYFORMAT_PASSARGS_12 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12 +#define TINYFORMAT_PASSARGS_13 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13 +#define TINYFORMAT_PASSARGS_14 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14 +#define TINYFORMAT_PASSARGS_15 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15 +#define TINYFORMAT_PASSARGS_16 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16 + +#define TINYFORMAT_PASSARGS_TAIL_1 +#define TINYFORMAT_PASSARGS_TAIL_2 , v2 +#define TINYFORMAT_PASSARGS_TAIL_3 , v2, v3 +#define TINYFORMAT_PASSARGS_TAIL_4 , v2, v3, v4 +#define TINYFORMAT_PASSARGS_TAIL_5 , v2, v3, v4, v5 +#define TINYFORMAT_PASSARGS_TAIL_6 , v2, v3, v4, v5, v6 +#define TINYFORMAT_PASSARGS_TAIL_7 , v2, v3, v4, v5, v6, v7 +#define TINYFORMAT_PASSARGS_TAIL_8 , v2, v3, v4, v5, v6, v7, v8 +#define TINYFORMAT_PASSARGS_TAIL_9 , v2, v3, v4, v5, v6, v7, v8, v9 +#define TINYFORMAT_PASSARGS_TAIL_10 , v2, v3, v4, v5, v6, v7, v8, v9, v10 +#define TINYFORMAT_PASSARGS_TAIL_11 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11 +#define TINYFORMAT_PASSARGS_TAIL_12 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12 +#define TINYFORMAT_PASSARGS_TAIL_13 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13 +#define TINYFORMAT_PASSARGS_TAIL_14 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14 +#define TINYFORMAT_PASSARGS_TAIL_15 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15 +#define TINYFORMAT_PASSARGS_TAIL_16 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16 + +#define TINYFORMAT_FOREACH_ARGNUM(m) \ + m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) m(10) m(11) m(12) m(13) m(14) m(15) m(16) +//[[[end]]] + + + +namespace detail { + +// Type-opaque holder for an argument to format(), with associated actions on +// the type held as explicit function pointers. This allows FormatArg's for +// each argument to be allocated as a homogenous array inside FormatList +// whereas a naive implementation based on inheritance does not. +class FormatArg +{ + public: + FormatArg() {} + + template + FormatArg(const T& value) + : m_value(static_cast(&value)), + m_formatImpl(&formatImpl), + m_toIntImpl(&toIntImpl) + { } + + void format(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc) const + { + m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); + } + + int toInt() const + { + return m_toIntImpl(m_value); + } + + private: + template + static void formatImpl(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value) + { + formatValue(out, fmtBegin, fmtEnd, ntrunc, *static_cast(value)); + } + + template + static int toIntImpl(const void* value) + { + return convertToInt::invoke(*static_cast(value)); + } + + const void* m_value; + void (*m_formatImpl)(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value); + int (*m_toIntImpl)(const void* value); +}; + + +// Parse and return an integer from the string c, as atoi() +// On return, c is set to one past the end of the integer. +inline int parseIntAndAdvance(const char*& c) +{ + int i = 0; + for(;*c >= '0' && *c <= '9'; ++c) + i = 10*i + (*c - '0'); + return i; +} + +// Print literal part of format string and return next format spec +// position. +// +// Skips over any occurrences of '%%', printing a literal '%' to the +// output. The position of the first % character of the next +// nontrivial format spec is returned, or the end of string. +inline const char* printFormatStringLiteral(std::ostream& out, const char* fmt) +{ + const char* c = fmt; + for(; true; ++c) + { + switch(*c) + { + case '\0': + out.write(fmt, static_cast(c - fmt)); + return c; + case '%': + out.write(fmt, static_cast(c - fmt)); + if(*(c+1) != '%') + return c; + // for "%%", tack trailing % onto next literal section. + fmt = ++c; + break; + } + } +} + + +// Parse a format string and set the stream state accordingly. +// +// The format mini-language recognized here is meant to be the one from C99, +// with the form "%[flags][width][.precision][length]type". +// +// Formatting options which can't be natively represented using the ostream +// state are returned in spacePadPositive (for space padded positive numbers) +// and ntrunc (for truncating conversions). argIndex is incremented if +// necessary to pull out variable width and precision . The function returns a +// pointer to the character after the end of the current format spec. +inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositive, + int& ntrunc, const char* fmtStart, + const detail::FormatArg* formatters, + int& argIndex, int numFormatters) +{ + if(*fmtStart != '%') + { + TINYFORMAT_ERROR("tinyformat: Not enough conversion specifiers in format string"); + return fmtStart; + } + // Reset stream state to defaults. + out.width(0); + out.precision(6); + out.fill(' '); + // Reset most flags; ignore irrelevant unitbuf & skipws. + out.unsetf(std::ios::adjustfield | std::ios::basefield | + std::ios::floatfield | std::ios::showbase | std::ios::boolalpha | + std::ios::showpoint | std::ios::showpos | std::ios::uppercase); + bool precisionSet = false; + bool widthSet = false; + int widthExtra = 0; + const char* c = fmtStart + 1; + // 1) Parse flags + for(;; ++c) + { + switch(*c) + { + case '#': + out.setf(std::ios::showpoint | std::ios::showbase); + continue; + case '0': + // overridden by left alignment ('-' flag) + if(!(out.flags() & std::ios::left)) + { + // Use internal padding so that numeric values are + // formatted correctly, eg -00010 rather than 000-10 + out.fill('0'); + out.setf(std::ios::internal, std::ios::adjustfield); + } + continue; + case '-': + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + continue; + case ' ': + // overridden by show positive sign, '+' flag. + if(!(out.flags() & std::ios::showpos)) + spacePadPositive = true; + continue; + case '+': + out.setf(std::ios::showpos); + spacePadPositive = false; + widthExtra = 1; + continue; + } + break; + } + // 2) Parse width + if(*c >= '0' && *c <= '9') + { + widthSet = true; + out.width(parseIntAndAdvance(c)); + } + if(*c == '*') + { + widthSet = true; + int width = 0; + if(argIndex < numFormatters) + width = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable width"); + if(width < 0) + { + // negative widths correspond to '-' flag set + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + width = -width; + } + out.width(width); + ++c; + } + // 3) Parse precision + if(*c == '.') + { + ++c; + int precision = 0; + if(*c == '*') + { + ++c; + if(argIndex < numFormatters) + precision = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable precision"); + } + else + { + if(*c >= '0' && *c <= '9') + precision = parseIntAndAdvance(c); + else if(*c == '-') // negative precisions ignored, treated as zero. + parseIntAndAdvance(++c); + } + out.precision(precision); + precisionSet = true; + } + // 4) Ignore any C99 length modifier + while(*c == 'l' || *c == 'h' || *c == 'L' || + *c == 'j' || *c == 'z' || *c == 't') + ++c; + // 5) We're up to the conversion specifier character. + // Set stream flags based on conversion specifier (thanks to the + // boost::format class for forging the way here). + bool intConversion = false; + switch(*c) + { + case 'u': case 'd': case 'i': + out.setf(std::ios::dec, std::ios::basefield); + intConversion = true; + break; + case 'o': + out.setf(std::ios::oct, std::ios::basefield); + intConversion = true; + break; + case 'X': + out.setf(std::ios::uppercase); + case 'x': case 'p': + out.setf(std::ios::hex, std::ios::basefield); + intConversion = true; + break; + case 'E': + out.setf(std::ios::uppercase); + case 'e': + out.setf(std::ios::scientific, std::ios::floatfield); + out.setf(std::ios::dec, std::ios::basefield); + break; + case 'F': + out.setf(std::ios::uppercase); + case 'f': + out.setf(std::ios::fixed, std::ios::floatfield); + break; + case 'G': + out.setf(std::ios::uppercase); + case 'g': + out.setf(std::ios::dec, std::ios::basefield); + // As in boost::format, let stream decide float format. + out.flags(out.flags() & ~std::ios::floatfield); + break; + case 'a': case 'A': + TINYFORMAT_ERROR("tinyformat: the %a and %A conversion specs " + "are not supported"); + break; + case 'c': + // Handled as special case inside formatValue() + break; + case 's': + if(precisionSet) + ntrunc = static_cast(out.precision()); + // Make %s print booleans as "true" and "false" + out.setf(std::ios::boolalpha); + break; + case 'n': + // Not supported - will cause problems! + TINYFORMAT_ERROR("tinyformat: %n conversion spec not supported"); + break; + case '\0': + TINYFORMAT_ERROR("tinyformat: Conversion spec incorrectly " + "terminated by end of string"); + return c; + } + if(intConversion && precisionSet && !widthSet) + { + // "precision" for integers gives the minimum number of digits (to be + // padded with zeros on the left). This isn't really supported by the + // iostreams, but we can approximately simulate it with the width if + // the width isn't otherwise used. + out.width(out.precision() + widthExtra); + out.setf(std::ios::internal, std::ios::adjustfield); + out.fill('0'); + } + return c+1; +} + + +//------------------------------------------------------------------------------ +inline void formatImpl(std::ostream& out, const char* fmt, + const detail::FormatArg* formatters, + int numFormatters) +{ + // Saved stream state + std::streamsize origWidth = out.width(); + std::streamsize origPrecision = out.precision(); + std::ios::fmtflags origFlags = out.flags(); + char origFill = out.fill(); + + for (int argIndex = 0; argIndex < numFormatters; ++argIndex) + { + // Parse the format string + fmt = printFormatStringLiteral(out, fmt); + bool spacePadPositive = false; + int ntrunc = -1; + const char* fmtEnd = streamStateFromFormat(out, spacePadPositive, ntrunc, fmt, + formatters, argIndex, numFormatters); + if (argIndex >= numFormatters) + { + // Check args remain after reading any variable width/precision + TINYFORMAT_ERROR("tinyformat: Not enough format arguments"); + return; + } + const FormatArg& arg = formatters[argIndex]; + // Format the arg into the stream. + if(!spacePadPositive) + arg.format(out, fmt, fmtEnd, ntrunc); + else + { + // The following is a special case with no direct correspondence + // between stream formatting and the printf() behaviour. Simulate + // it crudely by formatting into a temporary string stream and + // munging the resulting string. + std::ostringstream tmpStream; + tmpStream.copyfmt(out); + tmpStream.setf(std::ios::showpos); + arg.format(tmpStream, fmt, fmtEnd, ntrunc); + std::string result = tmpStream.str(); // allocates... yuck. + for(size_t i = 0, iend = result.size(); i < iend; ++i) + if(result[i] == '+') result[i] = ' '; + out << result; + } + fmt = fmtEnd; + } + + // Print remaining part of format string. + fmt = printFormatStringLiteral(out, fmt); + if(*fmt != '\0') + TINYFORMAT_ERROR("tinyformat: Too many conversion specifiers in format string"); + + // Restore stream state + out.width(origWidth); + out.precision(origPrecision); + out.flags(origFlags); + out.fill(origFill); +} + +} // namespace detail + + +/// List of template arguments format(), held in a type-opaque way. +/// +/// A const reference to FormatList (typedef'd as FormatListRef) may be +/// conveniently used to pass arguments to non-template functions: All type +/// information has been stripped from the arguments, leaving just enough of a +/// common interface to perform formatting as required. +class FormatList +{ + public: + FormatList(detail::FormatArg* formatters, int N) + : m_formatters(formatters), m_N(N) { } + + friend void vformat(std::ostream& out, const char* fmt, + const FormatList& list); + + private: + const detail::FormatArg* m_formatters; + int m_N; +}; + +/// Reference to type-opaque format list for passing to vformat() +typedef const FormatList& FormatListRef; + + +namespace detail { + +// Format list subclass with fixed storage to avoid dynamic allocation +template +class FormatListN : public FormatList +{ + public: +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + template + FormatListN(const Args&... args) + : FormatList(&m_formatterStore[0], N), + m_formatterStore{FormatArg(args)...} + { static_assert(sizeof...(args) == N, "Number of args must be N"); } +#else // C++98 version + void init(int) {} +# define TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR(n) \ + \ + template \ + FormatListN(TINYFORMAT_VARARGS(n)) \ + : FormatList(&m_formatterStore[0], n) \ + { assert(n == N); init(0, TINYFORMAT_PASSARGS(n)); } \ + \ + template \ + void init(int i, TINYFORMAT_VARARGS(n)) \ + { \ + m_formatterStore[i] = FormatArg(v1); \ + init(i+1 TINYFORMAT_PASSARGS_TAIL(n)); \ + } + + TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR) +# undef TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR +#endif + + private: + FormatArg m_formatterStore[N]; +}; + +// Special 0-arg version - MSVC says zero-sized C array in struct is nonstandard +template<> class FormatListN<0> : public FormatList +{ + public: FormatListN() : FormatList(0, 0) {} +}; + +} // namespace detail + + +//------------------------------------------------------------------------------ +// Primary API functions + +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + +/// Make type-agnostic format list from list of template arguments. +/// +/// The exact return type of this function is an implementation detail and +/// shouldn't be relied upon. Instead it should be stored as a FormatListRef: +/// +/// FormatListRef formatList = makeFormatList( /*...*/ ); +template +detail::FormatListN makeFormatList(const Args&... args) +{ + return detail::FormatListN(args...); +} + +#else // C++98 version + +inline detail::FormatListN<0> makeFormatList() +{ + return detail::FormatListN<0>(); +} +#define TINYFORMAT_MAKE_MAKEFORMATLIST(n) \ +template \ +detail::FormatListN makeFormatList(TINYFORMAT_VARARGS(n)) \ +{ \ + return detail::FormatListN(TINYFORMAT_PASSARGS(n)); \ +} +TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_MAKEFORMATLIST) +#undef TINYFORMAT_MAKE_MAKEFORMATLIST + +#endif + +/// Format list of arguments to the stream according to the given format string. +/// +/// The name vformat() is chosen for the semantic similarity to vprintf(): the +/// list of format arguments is held in a single function argument. +inline void vformat(std::ostream& out, const char* fmt, FormatListRef list) +{ + detail::formatImpl(out, fmt, list.m_formatters, list.m_N); +} + + +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + +/// Format list of arguments to the stream according to given format string. +template +void format(std::ostream& out, const char* fmt, const Args&... args) +{ + vformat(out, fmt, makeFormatList(args...)); +} + +/// Format list of arguments according to the given format string and return +/// the result as a string. +template +std::string format(const char* fmt, const Args&... args) +{ + std::ostringstream oss; + format(oss, fmt, args...); + return oss.str(); +} + +/// Format list of arguments to std::cout, according to the given format string +template +void printf(const char* fmt, const Args&... args) +{ + format(std::cout, fmt, args...); +} + + +#else // C++98 version + +inline void format(std::ostream& out, const char* fmt) +{ + vformat(out, fmt, makeFormatList()); +} + +inline std::string format(const char* fmt) +{ + std::ostringstream oss; + format(oss, fmt); + return oss.str(); +} + +inline void printf(const char* fmt) +{ + format(std::cout, fmt); +} + +#define TINYFORMAT_MAKE_FORMAT_FUNCS(n) \ + \ +template \ +void format(std::ostream& out, const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + vformat(out, fmt, makeFormatList(TINYFORMAT_PASSARGS(n))); \ +} \ + \ +template \ +std::string format(const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + std::ostringstream oss; \ + format(oss, fmt, TINYFORMAT_PASSARGS(n)); \ + return oss.str(); \ +} \ + \ +template \ +void printf(const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + format(std::cout, fmt, TINYFORMAT_PASSARGS(n)); \ +} + +TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMAT_FUNCS) +#undef TINYFORMAT_MAKE_FORMAT_FUNCS + +#endif + + +} // namespace tinyformat + +#endif // TINYFORMAT_H_INCLUDED diff --git a/cpp/include/unique_resource.hpp b/cpp/include/unique_resource.hpp new file mode 100644 index 0000000..56b3fe4 --- /dev/null +++ b/cpp/include/unique_resource.hpp @@ -0,0 +1,78 @@ +// https://gist.github.com/socantre/b45b3e23e6f1f4715f08 + +#pragma once + +#include + +namespace socantre { + +template class unique_resource { + R resource; + D deleter; + bool execute_on_destruction; // exposition only + unique_resource &operator=(unique_resource const &) = delete; + unique_resource(unique_resource const &) = delete; // no copies! + public: + // construction + explicit unique_resource(R &&resource, D &&deleter, + bool shouldrun = true) noexcept(true) + : resource(std::move(resource)), deleter(std::move(deleter)), + execute_on_destruction{shouldrun} {} + // move + unique_resource(unique_resource &&other) noexcept(true) + : resource(std::move(other.resource)), deleter(std::move(other.deleter)), + execute_on_destruction{other.execute_on_destruction} { + other.release(); + } + unique_resource & + operator=(unique_resource &&other) noexcept(noexcept(this->reset())) { + this->reset(); + this->deleter = std::move(other.deleter); + this->resource = std::move(other.resource); + this->execute_on_destruction = other.execute_on_destruction; + other.release(); + return *this; + } + // resource release + ~unique_resource() noexcept(noexcept(this->reset())) { this->reset(); } + void reset() noexcept(noexcept(this->get_deleter()(resource))) { + if (execute_on_destruction) { + this->execute_on_destruction = false; + this->get_deleter()(resource); + } + } + void reset(R &&newresource) noexcept(noexcept(this->reset())) { + this->reset(); + this->resource = std::move(newresource); + this->execute_on_destruction = true; + } + R const &release() noexcept(true) { + this->execute_on_destruction = false; + return this->get(); + } + // resource access + R const &get() const noexcept(true) { return this->resource; } + operator R const &() const noexcept(true) { return this->resource; } + R operator->() const noexcept(true) { return this->resource; } + std::add_lvalue_reference_t> operator*() const { + return *this->resource; + } + // deleter access + const D &get_deleter() const noexcept(true) { return this->deleter; } +}; +// factories +template +auto make_unique_resource(R &&r, D &&d) noexcept(true) +-> decltype(unique_resource>( + std::move(r), std::forward>(d), true)) { + return unique_resource>( + std::move(r), std::forward>(d), true); +} +template +auto make_unique_resource_checked(R r, R invalid, D d) noexcept(true) +-> decltype(unique_resource(std::move(r), std::move(d), true)) { + bool shouldrun = not bool(r == invalid); + return unique_resource(std::move(r), std::move(d), shouldrun); +} + +} // namespace socantre diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt new file mode 100644 index 0000000..cf8e5fa --- /dev/null +++ b/cpp/src/CMakeLists.txt @@ -0,0 +1,6 @@ +#For the shared library: +set ( PROJECT_LINK_LIBS liblmdb.so) +link_directories( /usr/local/lib) + +add_library(persipubsub SHARED queue.cpp queue.h library.h library.cpp filesystem.cpp) +target_link_libraries( persipubsub ${PROJECT_LINK_LIBS}) diff --git a/cpp/src/control.h b/cpp/src/control.h new file mode 100644 index 0000000..540c771 --- /dev/null +++ b/cpp/src/control.h @@ -0,0 +1,135 @@ +// +// Created by selim on 05.02.19. +// + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub { + namespace control { + /** + * Store queue parameter in lmdb. + * + * @param max_reader_num max number of reader of the lmdb + * @param max_db_num max number of named databases + * @param max_db_size_bytes max size of the lmdb in bytes + * @param env where parameters are stored + */ + void set_queue_parameters(int max_reader_num, int max_db_num, + int max_db_size_bytes, lmdb::env env); + + /** + * Set high water mark values for queue. + * + * @param hwm high water mark values + * @param env where values are stored + */ + void set_hwm(persipubsub::queue::HighWaterMark hwm, lmdb::env env); + + /** + * Set pruning strategy for queue. + * + * @param strategy pruning strategy + * @param env where strategy are stored + */ + void set_strategy(persipubsub::queue::Strategy strategy, lmdb::env env); + + /** + * Add a subscriber and create its lmdb. + * + * @param identifier ID of the subscriber which should be added + * @param env where subscriber is added + */ + void _add_sub(char identifier, lmdb::env env); + + /** + * Control and maintain one queue. + */ + class Control { + public: + /** + * Initialize control class. + * + * @param path to the queue. + */ + Control(boost::filesystem::path path) path_(path) {}; + + /** + * Initialize control with a (re)initialized queue. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_water_mark limit of the queue + * @param strategy used to prune queue + */ + void init(std::vector subscriber_ids, int max_readers, + int max_size, + persipubsub::queue::HighWaterMark high_water_mark, + persipubsub::queue::Strategy strategy); + + /** + * Check if queue is initialized. + * + * @return is initialized when all values for the given keys are set + */ + bool check_queue_is_initialized(); + + /** + * Clear all subscriber and delete all messages for queue. + */ + void clear_all_subscribers(); + + /** + * Prune all dangling messages from the lmdb. + */ + void prune_dangling_messages(); + + ~Control(); + private: + boost::filesystem::path path_; + persipubsub::queue::Queue queue_; + std::vector subscriber_ids_; + + /** + * Reinitialize the queue which is maintained by the control. + */ + void reinitialize_queue(); + + /** + * Initialize queue. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_water_mark limit of the queue + * @param strategy used to prune queue + */ + void initialize_queue(std::vector subscriber_ids, + int max_readers, int max_size, + persipubsub::queue::HighWaterMark high_water_mark, + persipubsub::queue::Strategy strategy); + + /** + * Prune all messages of a subscriber. + * + * @param identifier ID of the subscriber of which all messages should be pruned + */ + void prune_all_messages_for(std::vector identifier); + + /** + * Remove a subscriber and delete all its messages. + * + * @param identifier ID of the subscriber which should be removed + * @param env from where subscriber is removed + */ + void remove_sub(std::vector identifier, lmdb::env env); + + }; // class Control + } // namespace control +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/environment.h b/cpp/src/environment.h new file mode 100644 index 0000000..435ec1f --- /dev/null +++ b/cpp/src/environment.h @@ -0,0 +1,75 @@ +// +// Created by selim on 05.02.19. +// +#pragma once + +#include +#include +#include "queue.h" +#include "control.h" +#include "publisher.h" +#include "subscriber.h" + +namespace persipubsub{ + namespace environment{ + + /** + * Fabricate persipubsub components. + */ + class Environment{ + + public: + /** + * Initialize. + * + * @param path to the queue + */ + Environment(const boost::filesystem::path& path) : path_(path) {}; + + /** + * Fabricate a new control. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_watermark limit of the queue + * @param strategy used to prune queue + * @return Control to create and maintain queue + */ + persipubsub::control::Control new_control( + std::vector subscriber_ids, int max_readers, + int max_size, + persipubsub::queue::HighWaterMark high_watermark, + persipubsub::queue::Strategy strategy); + + /** + * Fabricate a new publisher. + * + * @param autosync if True, store data automatically in lmdb + * @return Publisher to send messages + */ + persipubsub::publisher::Publisher new_publisher(bool autosync); + + /** + * Fabricate a new subscriber. + * + * @param identifier of the subscriber + * @return Subscriber to receive messages + */ + persipubsub::subscriber::Subscriber new_subscriber(char identifier); + + private: + boost::filesystem::path path_; + + }; // class Environment + + /** + * Fabricate a new environment. + * + * @param path to the queue + * @return Environment to create control, publisher and subscriber + */ + persipubsub::environment::Environment new_environment(boost::filesystem::path path); + + } // namespace environment +} // namespace persipubsub diff --git a/cpp/src/filesystem.cpp b/cpp/src/filesystem.cpp new file mode 100644 index 0000000..05ee137 --- /dev/null +++ b/cpp/src/filesystem.cpp @@ -0,0 +1,317 @@ +// Copyright (c) 2016 Parquery AG. All rights reserved. +// Created by mristin on 10/12/16. + +#include "filesystem.h" + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +std::vector pqry::filesystem::ls_lt(const fs::path& directory) { + if (!fs::exists(directory)) { + throw std::invalid_argument(tfm::format("Directory must exist: %s", directory)); + } + + if (!fs::is_directory(directory)) { + throw std::invalid_argument( + tfm::format("Not a directory: %s %s", directory, fs::status(directory).type())); + } + + std::list > ts_pths; + for (const auto& entry : boost::make_iterator_range(fs::directory_iterator(directory), {})) { + struct stat st = {}; + stat(entry.path().c_str(), &st); + + // get the time in nanoseconds + const int64_t ns = pqry::filesystem::modified_time(entry.path()); + + ts_pths.emplace_back(std::make_tuple(ns, entry.path())); + } + + ts_pths.sort(); + + std::vector res; + res.reserve(ts_pths.size()); + + for (const auto& ts_pth : ts_pths) { + res.push_back(std::get<1>(ts_pth)); + } + + return res; +} + +std::vector pqry::filesystem::listdir(const fs::path& directory) { + if (not fs::exists(directory)) { + std::printf( "Directory does not exist"); + } + + if (not fs::is_directory(directory)) { + std::printf( "Not a directory"); + } + + std::vector res; + for (const auto& entry : boost::make_iterator_range(fs::directory_iterator(directory), {})) { + res.push_back(entry.path()); + } + return res; +} + +int64_t pqry::filesystem::modified_time(const fs::path& path) { + struct stat st = {}; + stat(path.c_str(), &st); + + // get the time in nanoseconds + const int64_t ns( + static_cast(st.st_mtim.tv_sec * 1000L * 1000L * 1000L) + static_cast(st.st_mtim.tv_nsec)); + + return ns; +} + +fs::path pqry::filesystem::mkdtemp() { + const fs::path tmpdir = fs::temp_directory_path() / fs::unique_path(); + fs::create_directories(tmpdir); + + return tmpdir; +} + +std::string pqry::filesystem::read(const fs::path& path) { + std::string content; + + std::ifstream ifs(path.c_str(), std::ios::in | std::ios::binary | std::ios::ate); + + if (ifs.good()) { + auto fsize = ifs.tellg(); + if (fsize < 0) { + std::printf( "File size retrieval failed"); + } + auto fsize_nonneg = static_cast(fsize); + + ifs.seekg(0, std::ios::beg); + + std::vector bytes(fsize_nonneg); + ifs.read(&bytes[0], fsize); + + if (ifs.good()) { + content = std::string(&bytes[0], fsize_nonneg); + } else { + std::printf( "error reading"); + } + } else { + std::printf( "error reading"); + } + + return content; +} + +void pqry::filesystem::write(const fs::path& path, const std::string& text) { + std::ofstream ofs(path.c_str(), std::ios::out | std::ios::binary); + ofs << text; + + if (!ofs.good()) { + std::printf( "error writing"); + } +} + +void pqry::filesystem::write(const fs::path& path, const char* bytes, int size) { + std::ofstream ofs(path.c_str(), std::ios::out | std::ios::binary); + + if (!ofs.good()) { + std::printf( "error writing"); + } + + ofs.write(bytes, size); +} + +void pqry::filesystem::copy_directory(const fs::path& source, const fs::path& dest) { + if (not fs::exists(source)) { + std::printf( "Error attempting to copy: Source does not exist"); + } + if (not fs::is_directory(source)) { + std::printf( "Error attempting to copy: Source is not a directory"); + } + if (not fs::exists(dest)) { + if (not fs::create_directories(dest)) { + std::printf( "Error attempting to create destination folder"); + } + } + + using rec_iterator = fs::recursive_directory_iterator; + for (auto it = rec_iterator(source, fs::symlink_option::no_recurse), end = rec_iterator(); it != end; ++it) { + // Remove common root + std::string difference = it->path().string(); + boost::replace_first(difference, source.string(), ""); + + if (not fs::exists(dest / difference)) { + boost::system::error_code error; + fs::copy(it->path(), dest / difference, error); + if (error.value() != 0) { + std::printf("Failed to copy"); + } + } else { + if (not fs::is_directory(it->path())) { + fs::copy_file(it->path(), dest / difference, fs::copy_option::overwrite_if_exists); + } + } + } +} + +void pqry::filesystem::unzip_archive(const fs::path& path, const fs::path& dest_dir) { + if (not fs::exists(path)) { + std::printf( "File does not exist"); + } + if (not fs::exists(dest_dir)) { + std::printf( "Expected the destination directory to exist"); + } + if (not fs::is_directory(dest_dir)) { + std::printf( "Not a directory"); + } + + // Read from the archive + // libzippp::ZipArchive zf(path.c_str()); + // zf.open(libzippp::ZipArchive::READ_ONLY); + // + // std::vector entries = zf.getEntries(); + // std::vector::iterator it; + // for (it = entries.begin(); it != entries.end(); ++it) { + // libzippp::ZipEntry entry = *it; + // + // // Create directory if entry is a dir + // if (entry.isDirectory()) { + // fs::path dir(dest_dir / entry.getName()); + // if (not fs::is_directory(dir.parent_path())) { + // fs::create_directory(dir.parent_path()); + // } + // continue; + // } + // + // // Only process if entry is a file + // if (not entry.isFile()) { + // continue; + // } + // + // fs::path entry_path(dest_dir / entry.getName()); + // std::ofstream ofs(entry_path.string()); + // entry.readContent(ofs); + // ofs.close(); + // } +} + +bool pqry::filesystem::LockFileGuard::lock(const fs::path& lock_file, int pid) { + if (locked_) { + unlock(); + } + + if (lock_file.empty()) { + std::printf(R"(lock_file path is "")"); + } + + fid_ = ::open(lock_file.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); + if (fid_ == -1) { + std::printf( "error opening the lock file for writing"); + } + + const int rc = ::flock(fid_, LOCK_EX | LOCK_NB); + + if (fid_ >= 0 and rc < 0) { + ::close(fid_); + locked_ = false; + return locked_; + } + + const std::string pid_str = tfm::format("%d", pid); + const ssize_t n = ::write(fid_, pid_str.c_str(), pid_str.size()); + if (n != pid_str.size()) { + std::printf( "error writing to lock file"); + } + + lock_file_ = lock_file; + locked_ = true; + return locked_; +} + +void pqry::filesystem::LockFileGuard::unlock() { + if (locked_) { + std::printf("Deleting lock file"); + ::close(fid_); + fs::remove(lock_file_); + } +} + +pqry::filesystem::LockFileGuard::~LockFileGuard() { + unlock(); +} + +void pqry::filesystem::wait_for_file(const fs::path& path, unsigned int timeout) { + int i = 0; + while (not fs::exists(path)) { + ::sleep(1); + ++i; + if (timeout > 0 and i > timeout) { + std::printf( "File does not exist (waited for some seconds)"); + } + } +} + +fs::path pqry::filesystem::expand_path(const fs::path& path) { + wordexp_t p = {}; + const int ret = wordexp(path.c_str(), &p, 0); + if (ret != 0) { + wordfree(&p); + std::printf( "Wordexp failed: %d", ret); + } + + fs::path expanded(p.we_wordv[0]); // NOLINT + wordfree(&p); + + return expanded; +} + +/** + * @param path to be added a suffix for a named temporary file + * @return path with a unique suffix + */ +fs::path with_temp_suffix(const fs::path& path) { + return path.parent_path() / + (path.stem().string() + "." + boost::filesystem::unique_path().string() + ".tmp" + path.extension().string()); +} + +pqry::filesystem::NamedTempfile::NamedTempfile( + const boost::filesystem::path& path) : pth_(path), tmp_pth_(with_temp_suffix(path)), renamed_(false) {} + +pqry::filesystem::NamedTempfile::~NamedTempfile() { + if (not renamed_) { + boost::system::error_code ec; // ignore all errors + fs::remove(tmp_pth_, ec); + } +} + +void pqry::filesystem::NamedTempfile::rename() { + if (renamed_) { + std::printf( "The temporary file has been already renamed"); + } + fs::rename(tmp_pth_, pth_); + renamed_ = true; +} diff --git a/cpp/src/library.cpp b/cpp/src/library.cpp new file mode 100644 index 0000000..796924e --- /dev/null +++ b/cpp/src/library.cpp @@ -0,0 +1,54 @@ +#include "library.h" + +#include + +namespace persipubsub { + unsigned int MAX_READER_NUM = 1024; + unsigned int MAX_DB_NUM = 1024; + unsigned long MAX_DB_SIZE_BYTES = 32UL * 1024UL * 1024UL * 1024UL; + + // define all database names here + char DATA_DB[] = "data_db"; // msg_id | data + char PENDING_DB[] = "pending_db"; // msg_id | pending subscriber + char META_DB[] = "meta_db"; // msg_id | metadata + char QUEUE_DB[] = "queue_db"; // queue_pth | all queue data + + char HWM_DB_SIZE_BYTES_KEY[] = "hwm_db_size_bytes"; + char MAX_MSGS_NUM_KEY[] = "max_msgs_num"; + char MSG_TIMEOUT_SECS_KEY[] = "msg_timeout_secs"; + char STRATEGY_KEY[] = "strategy"; + char SUBSCRIBER_IDS_KEY[] = "subscriber_ids"; + + QueueData lookup_queue_data(const lmdb::env& env) { + auto rtxn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + auto queue_dbi = lmdb::dbi::open(rtxn, QUEUE_DB); + + lmdb::val size_key(HWM_DB_SIZE_BYTES_KEY), size; + + queue_dbi.get(rtxn, size_key, size); + unsigned long hwm_db_size_bytes = std::stoul(size.data()); + + + lmdb::val msgs_key(MAX_MSGS_NUM_KEY), msgs_num; + queue_dbi.get(rtxn, msgs_key, msgs_num); + unsigned int max_msgs_num = std::stoi(msgs_num.data()); + + lmdb::val timeout_key(MSG_TIMEOUT_SECS_KEY), timeout; + queue_dbi.get(rtxn, timeout_key, timeout); + unsigned int msg_timeout_secs = std::stoi(timeout.data()); + + lmdb::val strategy_key(STRATEGY_KEY), strategy_val; + queue_dbi.get(rtxn, strategy_key, strategy_val); + std::string strategy_str = strategy_val.data(); + queue::Strategy strategy = queue::parse_strategy(strategy_str); + + lmdb::val subscriber_ids_key(SUBSCRIBER_IDS_KEY), subscriber_ids_val; + queue_dbi.get(rtxn, subscriber_ids_key, subscriber_ids_val); + std::string subscriber_ids_str = subscriber_ids_val.data(); + std::vector subscriber_ids; + + boost::split(subscriber_ids, subscriber_ids_str, boost::is_any_of(" "), boost::token_compress_on); + + return QueueData(msg_timeout_secs, max_msgs_num, hwm_db_size_bytes, strategy, subscriber_ids); + } +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/library.h b/cpp/src/library.h new file mode 100644 index 0000000..c032a9d --- /dev/null +++ b/cpp/src/library.h @@ -0,0 +1,50 @@ +// +// Created by selim on 01.02.19. +// + +#pragma once + +#include +#include +#include // Include boost::for is_any_of +#include // Include for boost::split +#include +#include "queue.h" + +namespace persipubsub { + // TODO read when it is appropriate to use extern + + extern unsigned int MAX_READER_NUM; + extern unsigned int MAX_DB_NUM; + extern unsigned long MAX_DB_SIZE_BYTES; + + // define all database names here + extern char DATA_DB[]; // msg_id | data + extern char PENDING_DB[]; // msg_id | pending subscriber + extern char META_DB[]; // msg_id | metadata + extern char QUEUE_DB[]; // queue_pth | all queue data + + extern char HWM_DB_SIZE_BYTES_KEY[]; + extern char MAX_MSGS_NUM_KEY[]; + extern char MSG_TIMEOUT_SECS_KEY[]; + extern char STRATEGY_KEY[]; + extern char SUBSCRIBER_IDS_KEY[]; + + struct QueueData{ + unsigned int msg_timeout_secs_; + unsigned int max_msgs_num_; + unsigned long hwm_db_size_bytes_; + persipubsub::queue::Strategy strategy_; + std::vector subscriber_ids_; + + QueueData(unsigned int msg_timeout_secs, unsigned int max_msgs_num, + unsigned long hwm_db_size_bytes, + persipubsub::queue::Strategy strategy, + std::vector subscriber_ids) : + msg_timeout_secs_(msg_timeout_secs), + max_msgs_num_(max_msgs_num), + hwm_db_size_bytes_(hwm_db_size_bytes), strategy_(strategy), + subscriber_ids_(std::move(subscriber_ids)) {}; + }; + QueueData lookup_queue_data(const lmdb::env& env); +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/publisher.h b/cpp/src/publisher.h new file mode 100644 index 0000000..bbeb108 --- /dev/null +++ b/cpp/src/publisher.h @@ -0,0 +1,52 @@ +// +// Created by selim on 05.02.19. +// + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub{ + namespace publisher{ + /** + * Create Publisher ready to send messages. + */ + class Publisher{ + + public: + Publisher() {}; + + /** + * Initialize. + * + * @param autosync sync after each message or after multiple messages + * @param path to the queue + */ + void init(bool autosync, boost::filesystem::path path); + + /** + * Send message to subscribers. + * + * @param msg to send to all subscribers + */ + void send(char msg); + + /** + * Send messages to subscribers. + * + * @param msgs to send to all subscribers + */ + void send_many(std::vector msgs); + + ~Publisher(); + private: + bool autosync_; + persipubsub::queue::Queue queue_; + + }; // class Publisher + } // namespace publisher +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/queue.cpp b/cpp/src/queue.cpp new file mode 100644 index 0000000..5119528 --- /dev/null +++ b/cpp/src/queue.cpp @@ -0,0 +1,410 @@ +// +// Created by selim on 13.02.19. +// + +// TODO nicer headers in files + +#include "queue.h" + +// +// Created by selim on 01.02.19. +// + +#include "library.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +std::map strategyMap; + +persipubsub::queue::Strategy persipubsub::queue::parse_strategy(const std::string &strategy) { + strategyMap["prune_first"] = persipubsub::queue::Strategy::prune_first; // TODO use consts for strings like prune_first - please google how (const staticexpr string????) + strategyMap["prune_last"] = persipubsub::queue::Strategy::prune_last; + + auto result = strategyMap.find(strategy); + if (result == strategyMap.end()) + throw std::runtime_error("Unknown strategy: " + strategy); + return result->second; + + +} + + +lmdb::env persipubsub::queue::initialize_environment(const fs::path& queue_dir, + const unsigned int max_reader_num, + const unsigned int max_db_num, + const unsigned long max_db_size_bytes) { + + if (not fs::exists(queue_dir)) + throw std::runtime_error("The queue directory does not exist: " + + queue_dir.filename().string()); + + auto env = lmdb::env::create(); + env.set_mapsize(max_db_size_bytes); + env.set_max_dbs(max_db_num); + env.set_max_readers(max_reader_num); + env.open(queue_dir.filename().string().c_str(), 0, 0664); + return env; + +} + + +void persipubsub::queue::prune_dangling_messages_for(const persipubsub::queue::Queue& queue, + const std::vector &subscriber_ids){ + + auto wtxn = lmdb::txn::begin(queue.env_); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + // Definition of dangling messages: + // - having no pending subscribers + // - exists longer than timeout allows + std::set msgs_to_delete; + + auto pending_cursor = lmdb::cursor::open(wtxn, pending_dbi); + + lmdb::val pending_key, pending_subscribers_num; + while (pending_cursor.get(pending_key, pending_subscribers_num, MDB_NEXT)) { + if (std::stoi(pending_subscribers_num.data()) == 0) + msgs_to_delete.insert(std::move(pending_key)); + } + pending_cursor.close(); + + time_t timer; + time(&timer); + const auto timestamp_now = boost::numeric_cast(timer); + + std::set msgs_to_delete_timeout; + + auto meta_cursor = lmdb::cursor::open(wtxn, pending_dbi); + + lmdb::val meta_key, timestamp; + while (meta_cursor.get(meta_key, timestamp, MDB_NEXT)) { + if (timestamp_now - std::stoi(timestamp.data()) > queue.hwm_.msg_timeout_secs_) + msgs_to_delete_timeout.insert(std::move(meta_key)); + } + meta_cursor.close(); + for (auto timeout_msg = msgs_to_delete_timeout.begin(); timeout_msg != msgs_to_delete_timeout.end(); ++timeout_msg){ + lmdb::val new_timeout_msg((*timeout_msg).data()); + msgs_to_delete.insert(std::move(new_timeout_msg)); + } + + for (auto delete_it = msgs_to_delete.begin(); delete_it != msgs_to_delete.end(); ++delete_it){ + pending_dbi.del(wtxn, *delete_it); + meta_dbi.del(wtxn, *delete_it); + data_dbi.del(wtxn, *delete_it); + } + + for (auto sub_it = queue.subscriber_ids_.begin(); sub_it != queue.subscriber_ids_.end(); ++sub_it){ + auto sub_dbi = lmdb::dbi::open(wtxn, (*sub_it).c_str()); + for (auto key_it = msgs_to_delete_timeout.begin(); key_it != msgs_to_delete_timeout.end(); ++key_it){ + sub_dbi.del(wtxn, *key_it); + } + } + + // todo replace with this style? YES + // for (const auto& sub_id : queue -> subscriber_ids_){ + // auto sub_dbi = lmdb::dbi::open(wtxn, sub_id); + // for (auto key_id : msgs_to_delete_timeout){ + // sub_dbi.del(wtxn, key_id); + // } + // } + + wtxn.commit(); +} + + +void persipubsub::queue::Queue::init(const boost::filesystem::path& path, lmdb::env env) { + path_ = path; + + if (env) + // todo check if move works YES + env_ = std::move(env); + else + env_ = persipubsub::queue::initialize_environment(path, persipubsub::MAX_READER_NUM, persipubsub::MAX_DB_NUM, persipubsub::MAX_DB_SIZE_BYTES); + // todo continue init + + { + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB, MDB_CREATE); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); + // lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); // TODO ask marko how to deal with it + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB, MDB_CREATE); + auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + + wtxn.commit(); // TODO this will not be needed, jumping out of scope + } + + + // todo lookup hwm data + + persipubsub::QueueData queue_data = persipubsub::lookup_queue_data(env_); + + hwm_ = HighWaterMark(queue_data.msg_timeout_secs_, + queue_data.max_msgs_num_, queue_data.hwm_db_size_bytes_); + + strategy_ = queue_data.strategy_; + + subscriber_ids_ = std::move(queue_data.subscriber_ids_); +} + + +void persipubsub::queue::Queue::put(const std::string& msg, const std::vector &subscriber_ids) const{ + + // todo uncomment + vacuum(); + // todo check UTC + time_t timer; + time(&timer); + const auto time = boost::lexical_cast(timer); + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + const std::string tmp = boost::uuids::to_string(uuid); + + // todo change all + std::string key = (time + tmp); + lmdb::val msg_id(key.c_str()); + + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + lmdb::val pending_subs((boost::lexical_cast(subscriber_ids.size())).c_str()); + pending_dbi.put(wtxn, msg_id, pending_subs); + + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + lmdb::val time_val(time); + meta_dbi.put(wtxn, msg_id, time_val); + + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + lmdb::val data_val(msg.c_str()); + data_dbi.put(wtxn, msg_id, data_val); + // todo change? + for (auto it = subscriber_ids.begin(); it != subscriber_ids.end(); ++it) { + auto sub_dbi = lmdb::dbi::open(wtxn, (*it).c_str()); + lmdb::val sub_data(""); + sub_dbi.put(wtxn, msg_id, sub_data); + } + + wtxn.commit(); + +} + +void persipubsub::queue::Queue::put_many_flush_once(const std::vector &msgs, + const std::vector &subscriber_ids) const { + vacuum(); + + // todo check UTC + time_t timer; + time(&timer); + const auto time = boost::lexical_cast(timer); + + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + // todo change? + std::vector sub_dbis; + for (auto it = subscriber_ids.begin(); it != subscriber_ids.end(); ++it) { + auto sub_dbi = lmdb::dbi::open(wtxn, (*it).c_str()); + sub_dbis.push_back(std::move(sub_dbi)); + } + + for (auto msg = msgs.begin(); msg != msgs.end(); ++msg) { + + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + const std::string tmp = boost::uuids::to_string(uuid); + std::string key = (time + tmp); + lmdb::val msg_id(key.c_str()); + + lmdb::val pending_subs((boost::lexical_cast(subscriber_ids.size())).c_str()); + pending_dbi.put(wtxn, msg_id, pending_subs); + lmdb::val time_val(time); + meta_dbi.put(wtxn, msg_id, time_val); + lmdb::val data_val(msg -> c_str()); + data_dbi.put(wtxn, msg_id, data_val); + + // todo change? + for (auto sub_dbi_it = sub_dbis.begin(); sub_dbi_it != sub_dbis.end(); ++sub_dbi_it) { + lmdb::val sub_data(""); + sub_dbi_it -> put(wtxn, msg_id, sub_data); + } + } + wtxn.commit(); + +} + +// todo msg as pointer or address? POINTER +void persipubsub::queue::Queue::front(const std::string& identifier, std::string* msg) const{ + + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto sub_dbi = lmdb::dbi::open(rtxn, identifier.c_str()); + auto cursor = lmdb::cursor::open(rtxn, sub_dbi); + lmdb::val key, value; + if (cursor.get(key, value, MDB_FIRST)) { + auto data_dbi = lmdb::dbi::open(rtxn, persipubsub::DATA_DB); + auto data_cursor = lmdb::cursor::open(rtxn, data_dbi); + lmdb::val data_value; + + bool found = data_cursor.get(key, data_value, MDB_FIRST); + if (found){ + std::string tmp_str = data_value.data(); + std::swap(*msg, tmp_str); + } + else{ + cursor.close(); + rtxn.abort(); + throw std::runtime_error("Data not found"); + } + } + cursor.close(); + rtxn.abort(); +} + +void persipubsub::queue::Queue::pop(const std::string& identifier) const { + auto wtxn = lmdb::txn::begin(env_); + auto sub_dbi = lmdb::dbi::open(wtxn, identifier.c_str()); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + + auto cursor = lmdb::cursor::open(wtxn, sub_dbi); + + lmdb::val key, value; + + if (cursor.get(key, value, MDB_FIRST)) { + sub_dbi.del(wtxn, key); + lmdb::val pending_value; + pending_dbi.get(wtxn, key, pending_value); + auto pending_num = boost::numeric_cast(std::stoi(pending_value.data())); + pending_num--; + lmdb::val pending_new_value((boost::lexical_cast(pending_num)).c_str()); + pending_dbi.put(wtxn, key, pending_new_value); + cursor.close(); + wtxn.commit(); + } + else { + cursor.close(); + wtxn.commit(); + throw std::runtime_error("No message to pop"); + } +} + + +void persipubsub::queue::Queue::prune_dangling_messages() const { + persipubsub::queue::prune_dangling_messages_for(*this, subscriber_ids_); +} + + +// todo size_t instead of long? +unsigned long persipubsub::queue::Queue::check_current_lmdb_size() const{ + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto data_dbi = lmdb::dbi::open(rtxn, persipubsub::DATA_DB); + + MDB_stat data_stat = data_dbi.stat(rtxn); + + // todo check overflow, boost::numeric_cast to unsigned long? + unsigned long lmdb_size_bytes = data_stat.ms_psize * (data_stat.ms_branch_pages + data_stat.ms_leaf_pages + data_stat.ms_overflow_pages); + + rtxn.abort(); + + return lmdb_size_bytes; + +} + + +unsigned int persipubsub::queue::Queue::count_msgs() const { + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto meta_dbi = lmdb::dbi::open(rtxn, persipubsub::META_DB); + + auto msgs_num = boost::numeric_cast(meta_dbi.stat(rtxn).ms_entries); + rtxn.abort(); + return msgs_num; +} + + +void persipubsub::queue::Queue::vacuum() const { + prune_dangling_messages(); + + unsigned int msgs_num = count_msgs(); + + if (msgs_num >= hwm_.max_msgs_num_) + prune_messages(); + unsigned long lmdb_size_bytes = check_current_lmdb_size(); + if (lmdb_size_bytes >= hwm_.hwm_lmdb_size_bytes_) + prune_messages(); +} + + +void persipubsub::queue::Queue::prune_messages() const { + + std::set messages_to_delete; + + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto rmeta_dbi = lmdb::dbi::open(rtxn, persipubsub::META_DB); + auto entries = boost::numeric_cast(rmeta_dbi.stat(rtxn).ms_entries); + + auto cursor = lmdb::cursor::open(rtxn, rmeta_dbi); + lmdb::val key, value; + + if (strategy_ == persipubsub::queue::Strategy::prune_first){ + cursor.get(key, value, MDB_FIRST); + for (unsigned int counter = 0; counter<=(entries/2); counter++){ + messages_to_delete.insert(std::move(key)); + cursor.get(key, value, MDB_NEXT); + } + } + else if (strategy_ == persipubsub::queue::Strategy::prune_last){ + cursor.get(key, value, MDB_LAST); + for (unsigned int counter = 0; counter<=(entries/2); counter++){ + messages_to_delete.insert(std::move(key)); + cursor.get(key, value, MDB_PREV); + } + } + else { + cursor.close(); + rtxn.abort(); + throw std::runtime_error("Pruning strategy not set."); + } + + cursor.close(); + rtxn.abort(); + + auto wtxn = lmdb::txn::begin(env_); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + std::vector dbis; + dbis.push_back(std::move(pending_dbi)); + dbis.push_back(std::move(meta_dbi)); + dbis.push_back(std::move(data_dbi)); + + // TODO iterate nicely, always try to use const auto& + for (auto sub_it = subscriber_ids_.begin(); sub_it != subscriber_ids_.end(); ++sub_it){ + auto sub_dbi = lmdb::dbi::open(wtxn, (*sub_it).c_str()); + dbis.push_back(std::move(sub_dbi)); + } + + for (auto msg_it = messages_to_delete.begin(); msg_it != messages_to_delete.end(); ++msg_it) { + for (auto db_it = dbis.begin(); db_it != dbis.end(); ++db_it){ + db_it -> del(wtxn, *msg_it); + } + } + + wtxn.commit(); +} + +persipubsub::queue::Queue::~Queue() {} + diff --git a/cpp/src/queue.h b/cpp/src/queue.h new file mode 100644 index 0000000..7780be5 --- /dev/null +++ b/cpp/src/queue.h @@ -0,0 +1,176 @@ +// +// Created by selim on 01.02.19. +// + +#pragma once + +#include +#include +#include + +namespace persipubsub { + namespace queue { + + class Queue; + + /** + * Store possible strategies. + */ + enum Strategy { + prune_first = 0, + prune_last + }; + + /** + * Parse overflow strategy. + * + * @param strategy Strategy stored in config + * @return set overflow strategy + */ + Strategy parse_strategy(const std::string& strategy); + + /** + * Store high water mark limits. + */ + struct HighWaterMark{ + /** + * Initialize. + * + * @param msg_timeout_secs time after which msg is classified as dangling msg (secs) + * @param max_masgs_num maximal amount of msg + * @param hwm_lmdb_size_bytes high water mark for total size of lmdb (bytes) + */ + unsigned int msg_timeout_secs_; // TODO set to default + unsigned int max_msgs_num_; + unsigned long hwm_lmdb_size_bytes_; + + HighWaterMark() : msg_timeout_secs_(), max_msgs_num_(), hwm_lmdb_size_bytes_() {} // TODO dont do this () + + HighWaterMark(const unsigned int msg_timeout_secs, + const unsigned int max_msgs_num, + const unsigned long hwm_lmdb_size) : msg_timeout_secs_(msg_timeout_secs), max_msgs_num_(max_msgs_num), hwm_lmdb_size_bytes_(hwm_lmdb_size) {} + }; + + /** + * Initialize the queue; the queue directory is assumed to exist. + * + * @param queue_dir where the queue is stored + * @param max_reader_num maximal number of reader + * @param max_db_num maximal number of databases + * @param max_db_size_bytes maximal size of database (bytes) + * @return Load or if needed create LMDB queue from directory + */ + lmdb::env initialize_environment(const boost::filesystem::path& queue_dir, const unsigned int max_reader_num, const unsigned int max_db_num, const unsigned long max_db_size_bytes); + + /** + * Prune all dangling messages for subscribers of a queue from lmdb. + * + * @param queue of which dangling messages should be pruned + * @param subscriber_ids subscribers of which dangling messages should be pruned + */ + void prune_dangling_messages_for(const persipubsub::queue::Queue& queue, const std::vector& subscriber_ids); + // todo string inside vector reference or normal? + /** + * Queue messages persistently from many publishers for many subscribers. + */ + class Queue{ + + public: + /** + * Initialize class object. + */ + Queue() : path_(), + env_(nullptr), hwm_(), strategy_(), + subscriber_ids_() {} + + /** + * Initialize the queue. + * + * @param config_pth path to the JSON config file + * @param queue_dir where the queue is stored + * @param max_reader_num maximal number of reader + * @param max_db_num maximal number of databases + * @param max_db_size_bytes maximal size of database (bytes) + */ + void init(const boost::filesystem::path& path, lmdb::env env= nullptr); + + /** + * Put message to lmdb queue. + * + * @param msg message send from publisher to subscribers + * @param subscriber_ids List of subscribers + */ + void put(const std::string& msg, const std::vector& subscriber_ids) const; // TODO make const functions const + // todo same string&? + /** + * Put many message to lmdb queue. + * + * @param msgs messages send from publisher to subscribers + * @param subscriber_ids List of subscribers + */ + void put_many_flush_once(const std::vector& msgs, const std::vector& subscriber_ids) const; + + /** + * Peek at next message in lmdb queue. + * + * Load from LMDB queue into memory and process msg afterwards. + * @param identifier Subscriber ID + * @param message peek on the message + */ + void front(const std::string& identifier, std::string* msg) const; + // todo change to pointer + /** + * Remove msg from the subscriber's queue and reduce pending subscribers. + * + * @param identifier Subscriber ID + */ + void pop(const std::string& identifier) const; + + /** + * Prune dangling messages in the queue. + */ + void prune_dangling_messages() const; + + /** + * Check current lmdb size in bytes. + * + * Check size of data database by approximating size with multiplying page size with number of pages. + * @return data database size in bytes + */ + unsigned long check_current_lmdb_size() const; + + /** + * Count number of messages in database. + * + * Count number of messages stored in meta database. + * + * @return number of messages in database + */ + unsigned int count_msgs() const; + + /** + * Clean database when needed. + */ + void vacuum() const; + + /** + * Prune one half of the messages stored. + * + * Depending on the strategy the first or the last will be deleted. + */ + void prune_messages() const; + + boost::filesystem::path path_; + lmdb::env env_; + HighWaterMark hwm_; + persipubsub::queue::Strategy strategy_; + std::vector subscriber_ids_; + + ~Queue(); + + private: + + }; // class Queue + + } // namespace queue +} // namespace persipubsub diff --git a/cpp/src/subscriber.h b/cpp/src/subscriber.h new file mode 100644 index 0000000..636ea9f --- /dev/null +++ b/cpp/src/subscriber.h @@ -0,0 +1,70 @@ +// +// Created by selim on 05.02.19. +// + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub{ + namespace subscriber{ + /** + * Create Subscriber ready to receive messages. + */ + class Subscriber{ + + public: + /** + * Initialize class object. + */ + Subscriber() {}; + + /** + * Initialize. + * + * @param identifier unique subscriber id + * @param path to the queue + */ + void init(std::vector identifier, boost::filesystem::path path); + + /** + * Receive messages from the publisher. + * + * @param timeout time waiting for a message. If none arrived until the timeout then None will be returned. (secs) + * @param retries number of tries to check if a msg arrived in the queue + * @return received message + */ + char receive(int timeout, int retries); + + /** + * Pops all messages until the most recent one and receive the latest. + * Used in the case that a particular subscriber cares only about the very + * last message and other subscribers care about all the messages in the + * queue. + * For another use case, when you only want to store the latest message + * and all subscribers are interested only in the latest, then use + * high water mark max_msgs_num = 1. + * + * @param timeout time waiting for a message. If none arrived until the timeout then None will be returned. (secs) + * @param retries number of tries to check if a msg arrived in the queue + * @return received message + */ + char receive_to_top(int timeout, int retries); + + ~Subscriber(); + private: + std::vector identifier_; + persipubsub::queue::Queue queue_; + + /** + * Pop a message from the subscriber's lmdb. + */ + void pop(); + + }; // class Subscriber + } // namespace subscriber +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt new file mode 100644 index 0000000..d1d1791 --- /dev/null +++ b/cpp/test/CMakeLists.txt @@ -0,0 +1,13 @@ +find_package (Boost COMPONENTS system filesystem unit_test_framework REQUIRED) +include_directories (${TEST_SOURCE_DIR}/src + ${Boost_INCLUDE_DIRS} + ) + +add_definitions (-DBOOST_TEST_DYN_LINK) +add_executable (Test test_queue.cpp) +target_link_libraries (Test + persipubsub + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} + ) \ No newline at end of file diff --git a/cpp/test/test_queue.cpp b/cpp/test/test_queue.cpp new file mode 100644 index 0000000..7f66b19 --- /dev/null +++ b/cpp/test/test_queue.cpp @@ -0,0 +1,281 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com and +// Selim Naji (selim.naji@parquery.com & marko@parquery.com) on 2019-02-15 + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MODULE TestQueue +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include "../src/queue.h" +#include "../src/library.h" + +namespace fs = boost::filesystem; + +class TmpDir{ +public: + TmpDir(){ + path_ = fs::unique_path(); + fs::create_directories(path_); + } + + ~TmpDir(){ + fs::remove_all(path_); + } + fs::path path_; +private: + +}; // class TmpDir + + +void setup_queue(const lmdb::txn& wtxn, const unsigned int msg_timeout_secs, + const unsigned int max_msgs_num, const unsigned long hwm_lmdb_size, + const std::string& strategy, const std::string& sub_ids){ + auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + + lmdb::val msg_timeout_secs_key(persipubsub::MSG_TIMEOUT_SECS_KEY); + lmdb::val msg_timeout_secs_val(boost::lexical_cast(msg_timeout_secs)); + queue_dbi.put(wtxn, msg_timeout_secs_key, msg_timeout_secs_val); + + lmdb::val max_msgs_num_key(persipubsub::MAX_MSGS_NUM_KEY); + lmdb::val max_msgs_num_val(boost::lexical_cast(max_msgs_num)); + queue_dbi.put(wtxn, max_msgs_num_key, max_msgs_num_val); + + lmdb::val hwm_lmdb_size_key(persipubsub::HWM_DB_SIZE_BYTES_KEY); + lmdb::val hwm_lmdb_size_val(boost::lexical_cast(hwm_lmdb_size)); + queue_dbi.put(wtxn, hwm_lmdb_size_key, hwm_lmdb_size_val); + + lmdb::val strategy_key(persipubsub::STRATEGY_KEY); + lmdb::val strategy_val(strategy); + queue_dbi.put(wtxn, strategy_key, strategy_val); + + lmdb::val subscriber_ids_key(persipubsub::SUBSCRIBER_IDS_KEY); + lmdb::val subscriber_ids_val(sub_ids); + queue_dbi.put(wtxn, subscriber_ids_key, subscriber_ids_val); +} + + +BOOST_AUTO_TEST_CASE(test_initialize_environment){ + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, 1024, 32UL * 1024UL * 1024UL * 1024UL); + + auto rtxn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + auto dbi = lmdb::dbi::open(rtxn, nullptr); + + MDB_envinfo info; + lmdb::env_info(env, &info); + + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(info.me_last_txnid)); + BOOST_REQUIRE_EQUAL("1", boost::lexical_cast(info.me_last_pgno)); + BOOST_REQUIRE_EQUAL("1024", boost::lexical_cast(info.me_maxreaders)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(info.me_mapaddr)); + BOOST_REQUIRE_EQUAL("34359738368", boost::lexical_cast(info.me_mapsize)); + BOOST_REQUIRE_EQUAL("1", boost::lexical_cast(info.me_numreaders)); + + MDB_stat stat = dbi.stat(rtxn); + + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_branch_pages)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_entries)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_depth)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_leaf_pages)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_overflow_pages)); + BOOST_REQUIRE_EQUAL("4096", boost::lexical_cast(stat.ms_psize)); + + rtxn.abort(); +} + +BOOST_AUTO_TEST_CASE(test_put_to_single_subscriber) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; // TODO read about carriage return, /0 and other funky issues + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + queue.put(msg, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + +} + +BOOST_AUTO_TEST_CASE(test_put_multiple_subscriber) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub1 sub2"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub1_dbi = lmdb::dbi::open(wtxn, "sub1", MDB_CREATE); + auto sub2_dbi = lmdb::dbi::open(wtxn, "sub2", MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.emplace_back("sub1"); + subs.emplace_back("sub2"); + queue.put(msg, subs); + + std::string result1; + queue.front("sub1", &result1); + BOOST_REQUIRE_EQUAL(msg, result1); + std::string result2; + queue.front("sub2", &result2); + BOOST_REQUIRE_EQUAL(msg, result2); +} + +BOOST_AUTO_TEST_CASE(test_put_many) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, + sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + + std::vector msgs; + unsigned int test_num_msgs = 10; + for (int num_msgs = test_num_msgs; num_msgs > 0; num_msgs--) + msgs.push_back(msg); + + queue.put_many_flush_once(msgs, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + BOOST_REQUIRE_EQUAL(test_num_msgs, queue.count_msgs()); +} + +BOOST_AUTO_TEST_CASE(test_pop) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, + sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + queue.put(msg, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + + queue.pop(sub_ids); + std::string empty; + queue.front(sub_ids, &empty); + BOOST_REQUIRE_EQUAL("", empty.c_str()); +} \ No newline at end of file diff --git a/py/.gitignore b/py/.gitignore new file mode 100644 index 0000000..0563776 --- /dev/null +++ b/py/.gitignore @@ -0,0 +1,65 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Sphinx documentation +docs/build/ + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv3/ +venv-doc/ + +# mypy +.mypy_cache/ + +# IDE +.idea + +# Archive +obsolete/ \ No newline at end of file diff --git a/py/.isort.cfg b/py/.isort.cfg new file mode 100644 index 0000000..822d029 --- /dev/null +++ b/py/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +known_first_party=persipubsub, tests \ No newline at end of file diff --git a/py/CHANGELOG.rst b/py/CHANGELOG.rst new file mode 100644 index 0000000..43bb01b --- /dev/null +++ b/py/CHANGELOG.rst @@ -0,0 +1,3 @@ +1.0.0 +===== +* Initial version \ No newline at end of file diff --git a/py/README.rst b/py/README.rst new file mode 100644 index 0000000..8c88e5b --- /dev/null +++ b/py/README.rst @@ -0,0 +1,364 @@ +persipubsub +=========== + +.. image:: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :target: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :alt: Build Status + +.. image:: https://coveralls.io/repos/github/Parquery/persipubsub/badge.svg?branch=master + :target: https://coveralls.io/github/Parquery/persipubsub?branch=master + :alt: Coverage + +.. image:: https://readthedocs.org/projects/persipubsub/badge/?version=latest + :target: https://persipubsub.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://badge.fury.io/py/persipubsub.svg + :target: https://badge.fury.io/py/persipubsub + :alt: PyPI - version + +.. image:: https://img.shields.io/pypi/pyversions/persipubsub.svg + :alt: PyPI - Python Version + +.. image:: https://badges.frapsoft.com/os/mit/mit.png?v=103 + :target: https://opensource.org/licenses/mit-license.php + :alt: MIT License + +``persipubsub`` implements a persistent, thread-safe and process-safe queue for +inter-process communication, based on `lmdb `_. + +Primarily, we used `zeromq `_ for inter-process +communication with a slight improvement through `persizmq +`_. This still did not fulfill the level +of persistence we wanted. + +Our motivation was to replace our previous library with a one which is +similarly easy to setup and to use. Additionally, it should make it possible to +send `protobuf `_ messages +(bytes) thread-safely and process-safely from many publishers to many +subscribers. + +Besides basic publisher and subscriber classes the library offers control +methods for easy deployment from a config JSON file and maintenance in case +needed. + +Related projects +================ + +persist-queue +------------- + +* The library offers not all functionality expected from a queue. It has put + and get function which are basically only push and pop. Therefore ``front`` + functionality is missing. In consequence neither can the queue have multiple + subscribers nor can be guaranteed that no data is lost when a thread fails. +* All messages in queues are serialized by ``pickle`` which was for us the main + reason not to use this library. + +Kafka +----- + +* Hence we only need Inter Process Communication, the TCP ability of `Kafka + `_ is an unnecessary overhead. +* Integration of ``Kafka`` written in Scala and Java in our C++/Python/Go + codebase is challenging. +* Hard to setup and maintain ``Kafka``. +* broker system eventually a bottleneck. + +RabbitMQ +-------- + +* Hence we only need Inter Process Communication, the TCP ability of `RabbitMQ + `_ is an unnecessary overhead. +* broker system eventually a bottleneck. +* ``RabbitMQ`` is less scalable than ``Kafka``, but + is supported officially by more languages. + +zeromq persistence pattern +-------------------------- + +* `Titanic `_ is the only persistence + pattern of `zeromq `_ which is also a broker system. + This takes away the purpose and advantage of ``zeromq`` to be a + lightweight library which requires no broker. + +Usage +===== + +The usage of the library consists of two steps: deployment and runtime + +Environment +----------- + +To improve the accessibility of the library, an environment class lets you +create and initialize any ``persipubsub`` component which you need in +deployment or runtime step. + +.. warning:: + + Only one environment of each queue per process allowed and it's forbidden to + fork environment or any child components to multiple processes. + In that case, persipubsub is multi-threading and multi-process safe. + If multiple environments of the same queue are active on the same process, + or environment is forked to multiple processes the lock is broken and + correctness can't be guaranteed. + +Initialize environment +^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + +Deployment +---------- + +In the deployment stage the library sets up the queue structure with the control. + +Control +^^^^^^^ + +A control unit to initialize and maintain queues. + +.. note:: + + The high water marks are limits for the queue. The message is deleted in + case that it reaches the timeout. In the other case of an overflow one + of two strategies is used to prune half of the queue. The choice is between + prune_first, which deletes the oldest messages, and prune_last, which + deletes the latest messages. + +Initialize queue +"""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + import persipubsub.queue + + env = persipubsub.environment.new_environment(path="/home/user/new-queue/") + + # Initialize a queue with default values. + control = env.new_control() + # Or define all optional parameters of the queue. + hwm = persipubsub.queue._HighWaterMark() + strategy = persipubsub.queue._Strategy.prune_first + control = env.new_control(subscriber_ids=["sub1", "sub2"], + high_watermark=hwm, + strategy=strategy) + +Prune all dangling messages +""""""""""""""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-dangling-messages/") + control = env.new_control() + + control.prune_dangling_messages() + +Clear all messages +"""""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-subscribers-and-messages/") + control = env.new_control() + + control.clear_all_subscribers() + + +Runtime +------- + +During runtime only publisher and subscriber are needed. + +.. note:: + + Control can be optionally be used for pruning although the queues prune + itself on a regular basis when high water mark is reached. The high water + mark includes a timeout, maximum number of messages and the maximum bytes + size of the queue. + +Publisher +^^^^^^^^^ + +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the publisher can +be initialized as following. + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + + pub = env.new_publisher() + +Send a message +"""""""""""""" + +.. code-block:: python + + msg = "Hello there!".encode('utf-8') + pub.send(msg=msg) + + # Subscribers have now a message in the queue. + +Send many messages at once +"""""""""""""""""""""""""" + +.. code-block:: python + + msgs = ["What's up?".encode('utf-8'), + "Do you like the README?".encode('utf-8')] + pub.send_many(msgs=msgs) + + # Both messages are now available for the subscribers. Note that the order + # of the messages are not necessarily kept. + +Subscriber +^^^^^^^^^^ + +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the subscriber can +be initialized as following. + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + + sub = env.new_subscriber(identifier="sub") + +Receive a message +""""""""""""""""" + +.. code-block:: python + + # One message in queue + with sub.receive() as msg: + # do something with the message + print(msg) # b'Hello there!' + + # This subscriber's queue is now empty + +Catch up with latest message +"""""""""""""""""""""""""""" + +Can be used in the case when a particular subscriber cares only about the very +last message. The messages are not popped for other subscribers. + +.. note:: + If you want to store only the latest message for all subscribers, then use + high water mark max_msgs_num = 1. + + +.. code-block:: python + + # Many outdated messages in queue + + with sub.receive_to_top() as msg: + # do something with the latest message + + # This subscriber's queue is now empty. + +Documentation +============= + +The documentation is available on `readthedocs +`_. + +Installation +============ + +* Install persipubsub with pip: + +.. code-block:: bash + + pip3 install persipubsub + +Development +=========== + +* Check out the repository. + +* In the repository root, create the virtual environment: + +.. code-block:: bash + + python3 -m venv venv3 + +* Activate the virtual environment: + +.. code-block:: bash + + source venv3/bin/activate + +* Install the development dependencies: + +.. code-block:: bash + + pip3 install -e .[dev] + +We use tox for testing and packaging the distribution. Assuming that the virtual +environment has been activated and the development dependencies have been +installed, run: + +.. code-block:: bash + + tox + +Pre-commit Checks +----------------- + +We provide a set of pre-commit checks that lint and check code for formatting. + +Namely, we use: + +* `yapf `_ to check the formatting. +* The style of the docstrings is checked with `pydocstyle `_. +* Static type analysis is performed with `mypy `_. +* `isort `_ to sort your imports for you. +* Various linter checks are done with `pylint `_. +* Doctests are executed using the Python `doctest module `_. +* `pyicontract-lint `_ lints contracts + in Python code defined with `icontract library `_. +* `twine `_ to check the README for invalid markup + which prevents it from rendering correctly on PyPI. + +Run the pre-commit checks locally from an activated virtual environment with +development dependencies: + +.. code-block:: bash + + ./precommit.py + +* The pre-commit script can also automatically format the code: + +.. code-block:: bash + + ./precommit.py --overwrite + +Versioning +========== + +We follow `Semantic Versioning `_. +The version X.Y.Z indicates: + +* X is the major version (backward-incompatible), +* Y is the minor version (backward-compatible), and +* Z is the patch version (backward-compatible bug fix). \ No newline at end of file diff --git a/py/docs/Makefile b/py/docs/Makefile new file mode 100644 index 0000000..ba501f6 --- /dev/null +++ b/py/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/py/docs/source/changelog.rst b/py/docs/source/changelog.rst new file mode 100644 index 0000000..ea1f942 --- /dev/null +++ b/py/docs/source/changelog.rst @@ -0,0 +1,5 @@ +********* +CHANGELOG +********* + +.. include:: ../../CHANGELOG.rst \ No newline at end of file diff --git a/py/docs/source/conf.py b/py/docs/source/conf.py new file mode 100644 index 0000000..b90d8c9 --- /dev/null +++ b/py/docs/source/conf.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) + +import persipubsub_meta + +# -- Project information ----------------------------------------------------- + +project = persipubsub_meta.__title__ +copyright = persipubsub_meta.__copyright__ +author = persipubsub_meta.__author__ +description = persipubsub_meta.__description__ + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = persipubsub_meta.__version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx_autodoc_typehints', + 'sphinx_icontract' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'persipubsubdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, + '{}.tex'.format(project), + '{} Documentation'.format(project), + author, + 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, + project, + '{} Documentation'.format(project), + [author], + 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, project, '{} Documentation'.format(project), + author, project, description, + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- diff --git a/py/docs/source/index.rst b/py/docs/source/index.rst new file mode 100644 index 0000000..08994e3 --- /dev/null +++ b/py/docs/source/index.rst @@ -0,0 +1,22 @@ +.. gswrap master file, created by + sphinx-quickstart on Wed Dec 5 15:14:21 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to persipubsub's documentation! +================================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + persipubsub/index.rst + readme + changelog + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` \ No newline at end of file diff --git a/py/docs/source/persipubsub/index.rst b/py/docs/source/persipubsub/index.rst new file mode 100644 index 0000000..f9bce07 --- /dev/null +++ b/py/docs/source/persipubsub/index.rst @@ -0,0 +1,12 @@ +########### +persipubsub +########### + +.. toctree:: + + persipubsub.ctl + persipubsub.env + persipubsub.queue + persipubsub.publisher + persipubsub.subscriber + persipubsub \ No newline at end of file diff --git a/py/docs/source/persipubsub/persipubsub.ctl.rst b/py/docs/source/persipubsub/persipubsub.ctl.rst new file mode 100644 index 0000000..65a03d6 --- /dev/null +++ b/py/docs/source/persipubsub/persipubsub.ctl.rst @@ -0,0 +1,5 @@ +Control +******* + +.. automodule:: persipubsub.control + :members: \ No newline at end of file diff --git a/py/docs/source/persipubsub/persipubsub.env.rst b/py/docs/source/persipubsub/persipubsub.env.rst new file mode 100644 index 0000000..a317995 --- /dev/null +++ b/py/docs/source/persipubsub/persipubsub.env.rst @@ -0,0 +1,5 @@ +Environment +*********** + +.. automodule:: persipubsub.environment + :members: \ No newline at end of file diff --git a/py/docs/source/persipubsub/persipubsub.publisher.rst b/py/docs/source/persipubsub/persipubsub.publisher.rst new file mode 100644 index 0000000..a6a6454 --- /dev/null +++ b/py/docs/source/persipubsub/persipubsub.publisher.rst @@ -0,0 +1,5 @@ +Publisher +********* + +.. automodule:: persipubsub.publisher + :members: diff --git a/py/docs/source/persipubsub/persipubsub.queue.rst b/py/docs/source/persipubsub/persipubsub.queue.rst new file mode 100644 index 0000000..3777523 --- /dev/null +++ b/py/docs/source/persipubsub/persipubsub.queue.rst @@ -0,0 +1,5 @@ +High Water Mark and Strategy +**************************** + +.. automodule:: persipubsub.queue + :members: \ No newline at end of file diff --git a/py/docs/source/persipubsub/persipubsub.rst b/py/docs/source/persipubsub/persipubsub.rst new file mode 100644 index 0000000..fc3a069 --- /dev/null +++ b/py/docs/source/persipubsub/persipubsub.rst @@ -0,0 +1,5 @@ +Tools +***** + +.. automodule:: persipubsub + :members: \ No newline at end of file diff --git a/py/docs/source/persipubsub/persipubsub.subscriber.rst b/py/docs/source/persipubsub/persipubsub.subscriber.rst new file mode 100644 index 0000000..dc53bdf --- /dev/null +++ b/py/docs/source/persipubsub/persipubsub.subscriber.rst @@ -0,0 +1,5 @@ +Subscriber +********** + +.. automodule:: persipubsub.subscriber + :members: diff --git a/py/docs/source/readme.rst b/py/docs/source/readme.rst new file mode 100644 index 0000000..787f7d6 --- /dev/null +++ b/py/docs/source/readme.rst @@ -0,0 +1,4 @@ +****** +README +****** +.. include:: ../../README.rst \ No newline at end of file diff --git a/py/mypy.ini b/py/mypy.ini new file mode 100644 index 0000000..57dfa98 --- /dev/null +++ b/py/mypy.ini @@ -0,0 +1,4 @@ +[mypy] + +[mypy-lmdb] +ignore_missing_imports = True \ No newline at end of file diff --git a/py/persipubsub/__init__.py b/py/persipubsub/__init__.py new file mode 100644 index 0000000..a57b115 --- /dev/null +++ b/py/persipubsub/__init__.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +"""Set default values of persipubsub and offers encoding tools.""" +from typing import List, Optional + +import lmdb + +# define all encodings here +ENCODING = 'utf-8' # type: str +BYTES_LENGTH = 8 # type: int +BYTES_ORDER = 'big' # type: str + +# define all database names here +DATA_DB = "data_db".encode(ENCODING) # msg_id | data +PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber +META_DB = "meta_db".encode(ENCODING) # msg_id | metadata +QUEUE_DB = "queue_db".encode(ENCODING) # queue_pth | all queue data + +# queues default +MAX_DB_SIZE_BYTES = 32 * 1024**3 # type: int +MAX_READER_NUM = 1024 # type: int +MAX_DB_NUM = 1024 # type: int + +# queues default keys +MAX_DB_SIZE_BYTES_KEY = "max_db_size_bytes".encode(ENCODING) +MAX_READER_NUM_KEY = "max_reader_num".encode(ENCODING) +MAX_DB_NUM_KEY = "max_db_num".encode(ENCODING) +HWM_DB_SIZE_BYTES_KEY = "hwm_db_size_bytes".encode(ENCODING) +MAX_MSGS_NUM_KEY = "max_msgs_num".encode(ENCODING) +MSG_TIMEOUT_SECS_KEY = "msg_timeout_secs".encode(ENCODING) +STRATEGY_KEY = "strategy".encode(ENCODING) +SUBSCRIBER_IDS_KEY = "subscriber_ids".encode(ENCODING) + + +def str_to_bytes(string: str) -> bytes: + """ + Encode a string with utf-8 encoding. + + :param string: any string + :return: string encoded with utf-8 + """ + return string.encode(encoding=ENCODING) + + +def bytes_to_str(encoded_str: bytes) -> str: + """ + Decode bytes with utf-8 encoding. + + :param encoded_str: any bytes + :return: bytes decoded with utf-8 + """ + return encoded_str.decode(encoding=ENCODING) + + +def int_to_bytes(value: int) -> bytes: + """ + Encode an integer to an array of bytes. + + :param value: any integer + :return: integer value representation as bytes + """ + return value.to_bytes(length=BYTES_LENGTH, byteorder=BYTES_ORDER) + + +def bytes_to_int(array_of_bytes: bytes) -> int: + """ + Decode an array of bytes to an integer. + + :param array_of_bytes: any array of bytes + :return: array of bytes representation as integer value + """ + return int.from_bytes(bytes=array_of_bytes, byteorder=BYTES_ORDER) + + +class QueueData: + """Hold queue data.""" + + def __init__(self, msg_timeout_secs: int, max_msgs_num: int, + hwm_db_size_bytes: int, strategy: str, + subscriber_ids: List[str]) -> None: + """ + Initialize. + + :param msg_timeout_secs: time after which msg is classified as dangling + msg (secs) + :param max_msgs_num: maximal amount of msg + :param hwm_db_size_bytes: high water mark for total size of LMDB (bytes) + :param strategy: pruning strategy + :param subscriber_ids: List of subscribers + """ + # pylint: disable=too-many-arguments + self.msg_timeout_secs = msg_timeout_secs + self.max_msgs_num = max_msgs_num + self.hwm_db_size_bytes = hwm_db_size_bytes + self.strategy = strategy + self.subscriber_ids = subscriber_ids + + +def lookup_queue_data(env: lmdb.Environment) -> QueueData: + """ + Lookup set value in named database 'queue_db'. + + Value stored in 'queue_db' are high water mark values and pruning strategy. + + :param env: environment that stores queue data + :return: lookup result + """ + with env.begin(write=False) as txn: + queue_db = env.open_db(key=QUEUE_DB, txn=txn, create=False) + msg_timeout_secs_bytes = txn.get(key=MSG_TIMEOUT_SECS_KEY, db=queue_db) + max_msgs_num_bytes = txn.get(key=MAX_MSGS_NUM_KEY, db=queue_db) + hwm_db_size_bytes = txn.get(key=HWM_DB_SIZE_BYTES_KEY, db=queue_db) + strategy_bytes = txn.get(key=STRATEGY_KEY, db=queue_db) + subscriber_ids_bytes = txn.get(key=SUBSCRIBER_IDS_KEY, db=queue_db) + + msg_timeout_secs = bytes_to_int(msg_timeout_secs_bytes) + max_msgs_num = bytes_to_int(max_msgs_num_bytes) + hwm_db_size = bytes_to_int(hwm_db_size_bytes) + + strategy = bytes_to_str(encoded_str=strategy_bytes) + + if subscriber_ids_bytes is None: + subscriber_ids = [] # type: List[str] + else: + subscriber_ids = bytes_to_str( + encoded_str=subscriber_ids_bytes).split(' ') + + queue_data = QueueData( + msg_timeout_secs=msg_timeout_secs, + max_msgs_num=max_msgs_num, + hwm_db_size_bytes=hwm_db_size, + strategy=strategy, + subscriber_ids=subscriber_ids) + return queue_data diff --git a/py/persipubsub/control.py b/py/persipubsub/control.py new file mode 100644 index 0000000..ecb9ea3 --- /dev/null +++ b/py/persipubsub/control.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Setup and control persistent queue.""" + +import pathlib +from typing import List, Optional, Sequence, Set + +import icontract +import lmdb # pylint: disable=unused-import + +import persipubsub.queue + +# pylint: disable=protected-access + + +class Control: + """Control and maintain a queue.""" + + def __init__(self, + path: pathlib.Path, + env: Optional[lmdb.Environment] = None): + """ + Initialize control class. + + :param path: to the queue. + :param env: open LMDB environment + """ + self.path = path + if not self.path.exists(): + self.path.mkdir(parents=True, exist_ok=True) + self.queue = None # type: Optional[persipubsub.queue._Queue] + self.subscriber_ids = set() # type: Set[str] + + if isinstance(env, lmdb.Environment): + self.env = env + else: + self.env = persipubsub.queue._initialize_environment( + queue_dir=self.path, + max_reader_num=persipubsub.MAX_READER_NUM, + max_db_num=persipubsub.MAX_DB_NUM, + max_db_size_bytes=persipubsub.MAX_DB_SIZE_BYTES) + + def init(self, + subscriber_ids: Optional[Sequence[str]] = None, + high_watermark: persipubsub.queue.HighWaterMark = persipubsub. + queue.HighWaterMark(), + strategy: persipubsub.queue.Strategy = persipubsub.queue.Strategy. + prune_first) -> None: + """ + Initialize control with a (re)initialized queue. + + :param subscriber_ids: subscribers of the queue + :param high_watermark: high water mark limit of the queue + :param strategy: used to prune queue + :return: + """ + # pylint: disable=too-many-arguments + if self.is_initialized(): + self._reinitialize_queue() + else: + if subscriber_ids is None: + subscriber_ids = [] + assert isinstance(subscriber_ids, Sequence) + self._initialize_queue( + subscriber_ids=subscriber_ids, + high_watermark=high_watermark, + strategy=strategy) + + def _reinitialize_queue(self) -> None: + """Reinitialize the queue which is maintained by the control.""" + self.queue = persipubsub.queue._Queue( + ) # type: persipubsub.queue._Queue + self.queue.init(path=self.path, env=self.env) + assert isinstance(self.queue.subscriber_ids, List) + self.subscriber_ids = set(self.queue.subscriber_ids) + + def _initialize_queue(self, + subscriber_ids: Sequence[str], + high_watermark: persipubsub.queue. + HighWaterMark = persipubsub.queue.HighWaterMark(), + strategy: persipubsub.queue.Strategy = persipubsub. + queue.Strategy.prune_first) -> None: + """ + Initialize queue. + + :param subscriber_ids: subscribers of the queue + :param high_watermark: high water mark limit of the queue + :param strategy: used to prune queue + :return: + """ + self.subscriber_ids = set(subscriber_ids) + # Databases needed for queue: + # 4 queues (data db, meta db, pending db, queue db) + # + each subscriber has its own db + + self.set_hwm(hwm=high_watermark) + self.set_strategy(strategy=strategy) + + for sub in self.subscriber_ids: + self._add_sub(sub_id=sub) + + # load initialized queue + self.queue = persipubsub.queue._Queue( + ) # type: persipubsub.queue._Queue + self.queue.init(path=self.path, env=self.env) + + def is_initialized(self) -> bool: + """ + Check if queue is initialized. + + :return: is initialized when all values for the given keys are set + """ + try: + _ = persipubsub.lookup_queue_data(env=self.env) + except lmdb.NotFoundError: + return False + + return True + + def clear_all_subscribers(self) -> None: + """Clear all subscriber and delete all messages for queue.""" + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.queue.env, lmdb.Environment) + with self.queue.env.begin(write=True) as txn: + for sub_id in self.subscriber_ids: + sub_db = self.queue.env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) + txn.drop(db=sub_db) + + pending_db = self.queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + txn.drop(db=pending_db, delete=False) + meta_db = self.queue.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + txn.drop(db=meta_db, delete=False) + data_db = self.queue.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + txn.drop(db=data_db, delete=False) + + def prune_dangling_messages(self) -> None: + """Prune all dangling messages from the LMDB.""" + assert isinstance(self.queue, persipubsub.queue._Queue) + persipubsub.queue._prune_dangling_messages_for( + queue=self.queue, subscriber_ids=list(self.subscriber_ids)) + + # pylint: disable=too-many-locals + def _prune_all_messages_for(self, sub_id: str) -> None: + """ + Prune all messages of a subscriber. + + :param sub_id: ID of the subscriber of which all messages should be + pruned + """ + msg_of_sub = set() + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.queue.env, lmdb.Environment) + with self.queue.env.begin(write=True) as txn: + sub_db = self.queue.env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) + + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + for key in cursor.iternext(keys=True, values=False): + msg_of_sub.add(key) + + txn.drop(db=sub_db, delete=False) + + pending_db = self.queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + for key in msg_of_sub: + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) + + @icontract.require(lambda sub_id: ' ' not in sub_id) + def _add_sub(self, sub_id: str) -> None: + """ + Add a subscriber and create its LMDB. + + :param sub_id: ID of the subscriber which should be added + """ + with self.env.begin(write=True) as txn: + _ = self.env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=True) + + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=True) + subscriber_ids = txn.get( + key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) + + if subscriber_ids is None: + subscriber_list = [] # type: List[str] + else: + subscriber_list = persipubsub.bytes_to_str( + encoded_str=subscriber_ids).split(' ') + subscriber_set = set(subscriber_list) + subscriber_set.add(sub_id) + subscriber_str = " ".join(subscriber_set) + txn.put( + key=persipubsub.SUBSCRIBER_IDS_KEY, + value=subscriber_str.encode(persipubsub.ENCODING), + db=queue_db) + + def _remove_sub(self, sub_id: str) -> None: + """ + Remove a subscriber and delete all its messages. + + :param sub_id: ID of the subscriber which should be removed + """ + msg_of_sub = set() + with self.env.begin(write=True) as txn: + sub_db = self.env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + for key in cursor.iternext(keys=True, values=False): + msg_of_sub.add(key) + txn.drop(db=sub_db) + + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + for key in msg_of_sub: + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) + + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=False) + subscriber_ids = txn.get( + key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) + + subscriber_list = persipubsub.bytes_to_str( + encoded_str=subscriber_ids).split(' ') + subscriber_set = set(subscriber_list) + subscriber_set.remove(sub_id) + subscriber_str = " ".join(subscriber_set) + txn.put( + key=persipubsub.SUBSCRIBER_IDS_KEY, + value=subscriber_str.encode(persipubsub.ENCODING), + db=queue_db) + + self.subscriber_ids.remove(sub_id) + + def set_hwm(self, hwm: persipubsub.queue.HighWaterMark) -> None: + """ + Set high water mark values for queue. + + :param hwm: high water mark values + :return: + """ + with self.env.begin(write=True) as txn: + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=True) + txn.put( + key=persipubsub.HWM_DB_SIZE_BYTES_KEY, + value=persipubsub.int_to_bytes(hwm.hwm_lmdb_size_bytes), + db=queue_db) + txn.put( + key=persipubsub.MAX_MSGS_NUM_KEY, + value=persipubsub.int_to_bytes(hwm.max_msgs_num), + db=queue_db) + txn.put( + key=persipubsub.MSG_TIMEOUT_SECS_KEY, + value=persipubsub.int_to_bytes(hwm.msg_timeout_secs), + db=queue_db) + + def set_strategy(self, strategy: persipubsub.queue.Strategy) -> None: + """ + Set pruning strategy for queue. + + :param strategy: pruning strategy + :param env: open LMDB environment + :return: + """ + with self.env.begin(write=True) as txn: + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=True) + txn.put( + key=persipubsub.STRATEGY_KEY, + value=persipubsub.str_to_bytes(str(strategy.name)), + db=queue_db) diff --git a/py/persipubsub/environment.py b/py/persipubsub/environment.py new file mode 100644 index 0000000..3f3e3c2 --- /dev/null +++ b/py/persipubsub/environment.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Create new persipubsub components.""" + +import pathlib +from typing import Any, Optional, Sequence + +import persipubsub.control +import persipubsub.publisher +import persipubsub.queue +import persipubsub.subscriber + +# pylint: disable = protected-access + + +class Environment: + """ + Create persipubsub components. + + Only one environment of each queue per process allowed and it's forbidden to + fork environment or any child components to multiple processes. + In that case, persipubsub is multi-threading and multi-process safe. + If multiple environments of the same queue are active on the same process, + or environment is forked to multiple processes the lock is broken and + correctness can't be guaranteed. + + :ivar path: to the queue + :vartype path: pathlib.Path + :ivar env: only open lmdb environment of process + :vartype env: lmdb.Environment + """ + + def __init__(self, path: pathlib.Path) -> None: + """ + Initialize. + + :param path: to the queue + """ + self.path = path + self.env = persipubsub.queue._initialize_environment( + queue_dir=self.path, + max_reader_num=persipubsub.MAX_READER_NUM, + max_db_num=persipubsub.MAX_DB_NUM, + max_db_size_bytes=persipubsub.MAX_DB_SIZE_BYTES) + + def __enter__(self) -> 'Environment': + """Enter the context and give environment prepared to constructor.""" + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Leave the context.""" + + # pylint: disable=too-many-arguments + def new_control(self, + subscriber_ids: Optional[Sequence[str]] = None, + high_watermark: persipubsub.queue. + HighWaterMark = persipubsub.queue.HighWaterMark(), + strategy: persipubsub.queue.Strategy = persipubsub.queue. + Strategy.prune_first) -> persipubsub.control.Control: + """ + Create a new control. + + :param subscriber_ids: subscribers of the queue + :param high_watermark: high water mark limit of the queue + :param strategy: used to prune queue + :return: Control to create and maintain queue + """ + control = persipubsub.control.Control(path=self.path, env=self.env) + control.init( + subscriber_ids=subscriber_ids, + high_watermark=high_watermark, + strategy=strategy) + return control + + def new_publisher( + self, autosync: bool = False) -> persipubsub.publisher.Publisher: + """ + Create a new publisher. + + :param autosync: if True, store data automatically in lmdb + :return: Publisher to send messages + """ + publisher = persipubsub.publisher.Publisher() + publisher.init(path=self.path, autosync=autosync, env=self.env) + return publisher + + def new_subscriber(self, + identifier: str) -> persipubsub.subscriber.Subscriber: + """ + Create a new subscriber. + + :param identifier: of the subscriber + :return: Subscriber to receive messages + """ + subscriber = persipubsub.subscriber.Subscriber() + subscriber.init(identifier=identifier, path=self.path, env=self.env) + return subscriber + + +def new_environment(path: pathlib.Path) -> Environment: + """ + Create a new environment. + + :param path: path to the queue + :return: Environment to create control, publisher and subscriber + """ + return Environment(path=path) diff --git a/py/persipubsub/publisher.py b/py/persipubsub/publisher.py new file mode 100644 index 0000000..22c77de --- /dev/null +++ b/py/persipubsub/publisher.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Publish messages to a queue and save them persistently.""" + +import pathlib +from typing import Any, List, Optional, Union + +import lmdb + +import persipubsub.queue + +# pylint: disable=protected-access + + +class Publisher: + """ + Handle publishing messages to the queue. + + :ivar queue: on which messages are published + :vartype queue: persipubsub.queue.Queue + :ivar autosync: sync after each message or after multiple messages + :vartype autosync: bool + """ + + def __init__(self) -> None: + """Initialize class object.""" + self.queue = None # type: Optional[persipubsub.queue._Queue] + self.autosync = None # type: Optional[bool] + + def init(self, + path: Union[pathlib.Path, str], + env: Optional[lmdb.Environment] = None, + autosync: bool = False) -> None: + """ + Initialize. + + :param path: path to the queue + :param env: open lmdb environment + :param autosync: if True, store data automatically in lmdb + """ + self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access + self.queue.init(path=path, env=env) + self.autosync = autosync + + def __enter__(self) -> 'Publisher': + """Enter the context and give the pub prepared in the constructor.""" + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Leave the context.""" + + def send(self, msg: bytes) -> None: + """ + Write one message to queue in one transaction. + + :param msg: to queue that all subscribers can read it + """ + assert isinstance(self.queue, persipubsub.queue._Queue) + self.queue.put(msg=msg) + + def send_many(self, msgs: List[bytes]) -> None: + """ + Write multiple messages to queue in one transaction. + + :param msgs: to queue that all subscribers can read them + """ + assert isinstance(self.queue, persipubsub.queue._Queue) + if self.autosync: + for msg in msgs: + self.queue.put(msg=msg) + else: + self.queue.put_many_flush_once(msgs=msgs) diff --git a/py/persipubsub/queue.py b/py/persipubsub/queue.py new file mode 100644 index 0000000..9d9234e --- /dev/null +++ b/py/persipubsub/queue.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +"""Store messages in a local LMDB.""" +import datetime +import enum +import pathlib +import uuid +from typing import Any, Dict, List, Optional, Union + +import lmdb + +import persipubsub + +MSG_TIMEOUT_SECS = 500 # type: int +MAX_MSGS_NUM = 1024 * 64 +HWM_LMDB_SIZE_BYTES = 30 * 1024**3 # type: int + + +class Strategy(enum.Enum): + """Hold possible strategies.""" + + prune_first = 0 + prune_last = 1 + + +def _parse_strategy(strategy: str) -> Strategy: + """ + Parse overflow strategy. + + :param strategy: Strategy stored in LMDB + :return: set overflow strategy + """ + if Strategy.prune_first.name == strategy: + return Strategy.prune_first + + if Strategy.prune_last.name == strategy: + return Strategy.prune_last + + raise ValueError("Unknown strategy: '{}' choosen.".format(strategy)) + + +class HighWaterMark: + """Hold high water mark limits.""" + + def __init__(self, + msg_timeout_secs: Optional[int] = None, + max_msgs_num: Optional[int] = None, + hwm_lmdb_size_bytes: Optional[int] = None) -> None: + """ + Initialize. + + :param msg_timeout_secs: time after which msg is classified as dangling + msg (secs) + :param max_msgs_num: maximal amount of msg + :param hwm_lmdb_size_bytes: high water mark for total size of LMDB + (bytes) + """ + self.msg_timeout_secs = MSG_TIMEOUT_SECS \ + if msg_timeout_secs is None else msg_timeout_secs # type: int + self.max_msgs_num = MAX_MSGS_NUM \ + if max_msgs_num is None else max_msgs_num # type: int + self.hwm_lmdb_size_bytes = HWM_LMDB_SIZE_BYTES \ + if hwm_lmdb_size_bytes is None else hwm_lmdb_size_bytes # type: int + + +def _initialize_environment( + queue_dir: pathlib.Path, + max_reader_num: int = 1024, + max_db_num: int = 1024, + max_db_size_bytes: int = 32 * 1024**3) -> lmdb.Environment: + """ + Initialize the queue; the queue directory is assumed to exist. + + :param queue_dir: where the queue is stored + :param max_reader_num: maximal number of readers + :param max_db_num: maximal number of databases + :param max_db_size_bytes: maximal size of database (bytes) + :return: Load or if needed create LMDB from directory + """ + if not queue_dir.exists(): + raise RuntimeError( + "The queue directory does not exist: {}".format(queue_dir)) + + env = lmdb.open( + path=queue_dir.as_posix(), + map_size=max_db_size_bytes, + subdir=True, + max_readers=max_reader_num, + max_dbs=max_db_num) + return env + + +def _prune_dangling_messages_for(queue: '_Queue', + subscriber_ids: List[str]) -> None: + """ + Prune all dangling messages for subscribers of a queue from LMDB. + + :param queue: of which dangling messages should be pruned + :param subscriber_ids: subscribers of which dangling msgs should be pruned + """ + assert isinstance(queue.env, lmdb.Environment) + with queue.env.begin(write=True) as txn: + pending_db = queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + meta_db = queue.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + data_db = queue.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + # Definition of dangling messages: + # - having no pending subscribers + # - exists longer than timeout allows + msgs_to_delete = set() + with queue.env.begin(db=pending_db) as txn: + cursor = txn.cursor() + cursor.first() + for key, pending_subscribers_num in cursor: + if persipubsub.bytes_to_int(pending_subscribers_num) == 0: + msgs_to_delete.add(key) + + # subscriber might still await these messages after the timeout. + # These messages needs also to be removed from all subscribers. + msgs_to_delete_timeout = set() + assert isinstance(queue.hwm, HighWaterMark) + with queue.env.begin(db=meta_db) as txn: + cursor = txn.cursor() + + timestamp_now = datetime.datetime.utcnow().timestamp() + for key, timestamp in cursor: + if int(timestamp_now) - persipubsub.bytes_to_int(timestamp) \ + > queue.hwm.msg_timeout_secs: + msgs_to_delete_timeout.add(key) + + msgs_to_delete = msgs_to_delete.union(msgs_to_delete_timeout) + with queue.env.begin(write=True) as txn: + for key in msgs_to_delete: + txn.delete(key=key, db=pending_db) + txn.delete(key=key, db=meta_db) + txn.delete(key=key, db=data_db) + + with queue.env.begin(write=True) as txn: + for sub_id in subscriber_ids: + sub_db = queue.env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) + for key in msgs_to_delete_timeout: + txn.delete(key=key, db=sub_db) + + +class _Queue: + """ + Represent a message queue. + + The messages come from many publishers and many subscribers. + They are stored persistently in an LMDB database. + + :ivar path: to the queue + :vartype config_pth: pathlib.Path + :ivar env: LMDB environment on disk + :vartype env: lmdb.Environment + :ivar hwm: High water mark limit for queue + :vartype hwm: HighWaterMark + :ivar strategy: + Strategy which will be used to remove messages when high water mark is + reached. + :vartype strategy: Strategy + :ivar subscriber_ids: all subscribers of the queue + :vartype sub_list: List[str] + """ + + def __init__(self) -> None: + """Initialize class object.""" + self.path = None # type: Optional[pathlib.Path] + self.env = None # type: Optional[lmdb.Environment] + self.hwm = None # type: Optional[HighWaterMark] + self.strategy = None # type: Optional[Strategy] + self.subscriber_ids = None # type: Optional[List[str]] + + def init(self, + path: Union[pathlib.Path, str], + env: Optional[lmdb.Environment] = None) -> None: + """ + Initialize the queue. + + :param path: where the queue is stored + :param env: open LMDB environment + """ + # pylint: disable=too-many-arguments + # pylint: disable=too-many-locals + self.path = path if isinstance(path, pathlib.Path) \ + else pathlib.Path(path) + + if isinstance(env, lmdb.Environment): + self.env = env + else: + self.env = _initialize_environment( + queue_dir=self.path, + max_reader_num=persipubsub.MAX_READER_NUM, + max_db_num=persipubsub.MAX_DB_NUM, + max_db_size_bytes=persipubsub.MAX_DB_SIZE_BYTES) + + with self.env.begin(write=True) as txn: + _ = self.env.open_db(key=persipubsub.DATA_DB, txn=txn, create=True) + _ = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=True) + _ = self.env.open_db(key=persipubsub.META_DB, txn=txn, create=True) + _ = self.env.open_db(key=persipubsub.QUEUE_DB, txn=txn, create=True) + + queue_data = persipubsub.lookup_queue_data(env=self.env) + + self.hwm = HighWaterMark( + msg_timeout_secs=queue_data.msg_timeout_secs, + max_msgs_num=queue_data.max_msgs_num, + hwm_lmdb_size_bytes=queue_data.hwm_db_size_bytes) + + self.strategy = _parse_strategy(strategy=queue_data.strategy) + + self.subscriber_ids = queue_data.subscriber_ids + + def __enter__(self) -> '_Queue': + """Enter the context and give the queue prepared in the constructor.""" + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Leave the context.""" + + def put(self, msg: bytes) -> None: + """ + Put message to LMDB in one transaction. + + :param msg: message in bytes + :return: + """ + # every publisher always prunes queue before sending a message. + self.vacuum() + msg_id = str(datetime.datetime.utcnow().timestamp()) + str(uuid.uuid4()) + assert isinstance(self.env, lmdb.Environment) + assert isinstance(self.subscriber_ids, List) + with self.env.begin(write=True) as txn: + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + txn.put( + key=persipubsub.str_to_bytes(msg_id), + value=persipubsub.int_to_bytes(len(self.subscriber_ids)), + db=pending_db) + + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + txn.put( + key=persipubsub.str_to_bytes(msg_id), + value=persipubsub.int_to_bytes( + int(datetime.datetime.utcnow().timestamp())), + db=meta_db) + + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + txn.put(key=persipubsub.str_to_bytes(msg_id), value=msg, db=data_db) + + for sub in self.subscriber_ids: + sub_db = self.env.open_db( + key=persipubsub.str_to_bytes(sub), txn=txn, create=False) + txn.put(key=persipubsub.str_to_bytes(msg_id), db=sub_db) + + def put_many_flush_once(self, msgs: List[bytes]) -> None: + """ + Put multiple message to LMDB in one transaction. + + :param msgs: messages in bytes + :return: + """ + # every publisher always prunes queue before sending a message. + self.vacuum() + assert isinstance(self.env, lmdb.Environment) + assert isinstance(self.subscriber_ids, List) + with self.env.begin(write=True) as txn: + + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + sub_dbs = set() + for sub in self.subscriber_ids: + sub_dbs.add( + self.env.open_db( + key=persipubsub.str_to_bytes(sub), + txn=txn, + create=False)) + + for msg in msgs: + msg_id = str(datetime.datetime.utcnow().timestamp()) + str( + uuid.uuid4()) + + txn.put( + key=persipubsub.str_to_bytes(msg_id), + value=persipubsub.int_to_bytes(len(self.subscriber_ids)), + db=pending_db) + + txn.put( + key=persipubsub.str_to_bytes(msg_id), + value=persipubsub.int_to_bytes( + int(datetime.datetime.utcnow().timestamp())), + db=meta_db) + + txn.put( + key=persipubsub.str_to_bytes(msg_id), value=msg, db=data_db) + + for sub_db in sub_dbs: + txn.put(key=persipubsub.str_to_bytes(msg_id), db=sub_db) + + def front(self, identifier: str) -> Optional[bytes]: + """ + Peek at next message in LMDB. + + Load from LMDB into memory and process msg afterwards. + + :param identifier: Subscriber ID + :return: + """ + assert isinstance(self.env, lmdb.Environment) + with self.env.begin(write=False) as txn: + sub_db = self.env.open_db( + key=persipubsub.str_to_bytes(identifier), txn=txn, create=False) + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + key = cursor.key() + msg = txn.get(key=key, db=data_db) + else: + msg = None + + return msg # type: ignore + + def pop(self, identifier: str) -> None: + """ + Remove msg from the subscriber's queue and reduce pending subscribers. + + :param identifier: Subscriber ID + :return: + """ + assert isinstance(self.env, lmdb.Environment) + with self.env.begin(write=True) as txn: + sub_db = self.env.open_db( + key=persipubsub.str_to_bytes(identifier), txn=txn, create=False) + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + key = cursor.key() + cursor.pop(key=key) + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) + else: + raise RuntimeError("No message to pop") + + def prune_dangling_messages(self) -> None: + """ + Prune dangling messages in the queue. + + Definition of dangling messages: + - having no pending subscribers + - exists longer than timeout allows + :return: + """ + assert isinstance(self.subscriber_ids, List) + _prune_dangling_messages_for( + queue=self, subscriber_ids=self.subscriber_ids) + + def check_current_lmdb_size(self) -> int: + """ + Check current LMDB size in bytes. + + Check size of data database by approximating size with multiplying page + size with number of pages. + + :return: data database size in bytes + """ + lmdb_size_bytes = 0 + assert isinstance(self.env, lmdb.Environment) + with self.env.begin(write=False) as txn: + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + data_stat = txn.stat(db=data_db) + lmdb_size_bytes += data_stat['psize'] * ( + data_stat['branch_pages'] + data_stat['leaf_pages'] + + data_stat['overflow_pages']) + + return lmdb_size_bytes + + def count_msgs(self) -> int: + """ + Count number of messages in database. + + Count number of messages stored in named database 'meta_db'. + + :return: number of messages in database + """ + assert isinstance(self.env, lmdb.Environment) + with self.env.begin(write=False) as txn: + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + meta_stat = txn.stat(db=meta_db) # type: Dict[str, int] + + return meta_stat['entries'] + + def vacuum(self) -> None: + """ + Clean database when needed. + + :return: + """ + assert isinstance(self.hwm, HighWaterMark) + self.prune_dangling_messages() + msgs_num = self.count_msgs() + if msgs_num >= self.hwm.max_msgs_num: + self.prune_messages() + lmdb_size_bytes = self.check_current_lmdb_size() + if lmdb_size_bytes >= self.hwm.hwm_lmdb_size_bytes: + self.prune_messages() + + def prune_messages(self) -> None: + """ + Prune one half of the messages stored. + + Depending on the strategy the first or the last will be deleted. + + :return: + """ + # pylint: disable=too-many-locals + messages_to_delete = set() + assert isinstance(self.env, lmdb.Environment) + with self.env.begin(write=False) as txn: + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + meta_stat = txn.stat(db=meta_db) + entries = meta_stat['entries'] + + cursor = txn.cursor(db=meta_db) + if self.strategy == Strategy.prune_first: + + cursor.first() + for index, key in enumerate( + cursor.iternext(keys=True, values=False)): + messages_to_delete.add(key) + if index >= int(entries / 2): + break + + elif self.strategy == Strategy.prune_last: + cursor.last() + for index, key in enumerate( + cursor.iterprev(keys=True, values=False)): + messages_to_delete.add(key) + if index >= int(entries / 2): + break + else: + raise RuntimeError("Pruning strategy not set.") + + assert isinstance(self.subscriber_ids, List) + with self.env.begin(write=True) as txn: + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + dbs = [pending_db, meta_db, data_db] + + for sub in self.subscriber_ids: + sub_db = self.env.open_db( + key=persipubsub.str_to_bytes(sub), txn=txn, create=False) + dbs.append(sub_db) + + for key in messages_to_delete: + for db in dbs: # pylint: disable=invalid-name + txn.delete(key=key, db=db) diff --git a/py/persipubsub/subscriber.py b/py/persipubsub/subscriber.py new file mode 100644 index 0000000..53b298d --- /dev/null +++ b/py/persipubsub/subscriber.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Receive messages persistently from the queue.""" + +import contextlib +import datetime +import pathlib +import time +from typing import Any, Iterator, Optional, Union + +import icontract +import lmdb # pylint: disable=unused-import + +import persipubsub.queue + +# pylint: disable=protected-access + + +class Subscriber: + """ + Handle receiving messages stored in the queue. + + :ivar identifier: subscriber ID + :vartype pub_id: str + :ivar queue: from which messages are received + :vartype queue: persipubsub.queue.Queue + """ + + def __init__(self) -> None: + """Initialize class object.""" + self.identifier = None # type: Optional[str] + self.queue = None # type: Optional[persipubsub.queue._Queue] + + def init(self, + identifier: str, + path: Union[pathlib.Path, str], + env: Optional[lmdb.Environment] = None) -> None: + """ + Initialize. + + :param identifier: unique subscriber id + :param path: path to the queue + :param env: open lmdb environment + """ + self.identifier = identifier + assert isinstance(self.identifier, str) + self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access + self.queue.init(path=path, env=env) + assert isinstance(self.queue, persipubsub.queue._Queue) + + def __enter__(self) -> 'Subscriber': + """Enter the context and give the sub prepared in the constructor.""" + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Leave the context.""" + + @icontract.require(lambda timeout: timeout > 0) + @icontract.require(lambda retries: retries > 0) + @contextlib.contextmanager + def receive(self, timeout: int = 60, + retries: int = 10) -> Iterator[Optional[bytes]]: + """ + Receive messages from the queue. + + :param timeout: time waiting for a message. If none arrived until the + timeout then None will be returned. (secs) + :param retries: number of tries to check if a msg arrived in the queue + """ + msg = None + end = int(datetime.datetime.utcnow().timestamp()) + timeout + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.identifier, str) + try: + while int(datetime.datetime.utcnow().timestamp()) <= end: + msg = self.queue.front(identifier=self.identifier) + if msg is not None: + break + time.sleep(timeout / retries) + yield msg + finally: + pass + + if msg is not None: + self._pop() + + def _pop(self) -> None: + """Pop a message from the subscriber's lmdb.""" + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.identifier, str) + self.queue.pop(identifier=self.identifier) + + @icontract.require(lambda timeout: timeout > 0) + @icontract.require(lambda retries: retries > 0) + @contextlib.contextmanager + def receive_to_top(self, timeout: int = 60, + retries: int = 10) -> Iterator[Optional[bytes]]: + """ + Pops all messages until the most recent one and receive the latest. + + Can be used in the case when a particular subscriber cares only about + the very last message. The messages are not popped for other + subscribers. + + If you want to store only the latest message for all subscribers, then + use high water mark max_msgs_num = 1. + + :param timeout: time waiting for a message. If none arrived until the + timeout then None will be returned. (secs) + :param retries: number of tries to check if a msg arrived in the queue + """ + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.queue.env, lmdb.Environment) + assert isinstance(self.identifier, str) + with self.queue.env.begin(write=False) as txn: + sub_db = self.queue.env.open_db( + key=persipubsub.str_to_bytes(self.identifier), + txn=txn, + create=False) + sub_stat = txn.stat(db=sub_db) + # pop all message except the most recent one + msg_to_pop_num = sub_stat['entries'] - 1 + + for _ in range(msg_to_pop_num): + self.queue.pop(identifier=self.identifier) + + msg = None + end = int(datetime.datetime.utcnow().timestamp()) + timeout + try: + while int(datetime.datetime.utcnow().timestamp()) <= end: + msg = self.queue.front(identifier=self.identifier) + if msg is not None: + break + time.sleep(timeout / retries) + yield msg + finally: + pass + + if msg is not None: + self._pop() diff --git a/py/persipubsub_meta.py b/py/persipubsub_meta.py new file mode 100644 index 0000000..0ed375a --- /dev/null +++ b/py/persipubsub_meta.py @@ -0,0 +1,10 @@ +"""Define meta information about persipubsub package.""" + +__title__ = 'persipubsub' +__description__ = 'Implements a persistent, thread-safe and process-safe queue for inter-process communication.' +__url__ = 'https://github.com/Parquery/persipubsub' +__version__ = '1.0.0' +__author__ = 'Selim Naji, Adam Radomski and Marko Ristin' +__author_email__ = 'selim.naji@parquery.com, adam.radomski@parquery.com, marko.ristin@gmail.com' +__license__ = 'MIT' +__copyright__ = 'Copyright 2019 Parquery AG' \ No newline at end of file diff --git a/py/precommit.py b/py/precommit.py new file mode 100644 index 0000000..6bbd70f --- /dev/null +++ b/py/precommit.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Runs precommit checks on the repository.""" +import argparse +import os +import pathlib +import subprocess +import sys +from typing import List # pylint: disable=unused-imports + + +def main() -> int: + """" + Main routine + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + help= + "Overwrites the unformatted source files with the well-formatted code in place. " + "If not set, an exception is raised if any of the files do not conform to the style guide.", + action='store_true') + + args = parser.parse_args() + + overwrite = bool(args.overwrite) + + repo_root = pathlib.Path(__file__).parent + + print("YAPF'ing...") + if overwrite: + subprocess.check_call([ + "yapf", "--in-place", "--style=style.yapf", "--recursive", "tests", + "persipubsub", "setup.py", "precommit.py" + ], + cwd=repo_root.as_posix()) + else: + subprocess.check_call([ + "yapf", "--diff", "--style=style.yapf", "--recursive", "tests", + "persipubsub", "bin", "setup.py", "precommit.py" + ], + cwd=repo_root.as_posix()) + + print("Mypy'ing...") + subprocess.check_call(["mypy", "--strict", "persipubsub", "tests"], + cwd=repo_root.as_posix()) + + print("Isort'ing...") + isort_files = [] # type: List[str] + for path in (repo_root / "persipubsub").glob("**/*.py"): + isort_files.append(path.as_posix()) + for path in (repo_root / "tests").glob("**/*.py"): + isort_files.append(path.as_posix()) + + if overwrite: + cmd = [ + "isort", "--balanced", "--multi-line", "4", "--line-width", "80", + "--dont-skip", "__init__.py", "--project", "persipubsub" + ] + cmd.extend(isort_files) + subprocess.check_call(cmd) + else: + cmd = [ + "isort", "--check-only", "--diff", "--balanced", "--multi-line", + "4", "--line-width", "80", "--dont-skip", "__init__.py", + "--project", "persipubsub" + ] + cmd.extend(isort_files) + subprocess.check_call(cmd) + + print("Pylint'ing...") + subprocess.check_call( + ["pylint", "--rcfile=pylint.rc", "tests", "persipubsub"], + cwd=repo_root.as_posix()) + + print("Pydocstyle'ing...") + subprocess.check_call(["pydocstyle", "persipubsub"], + cwd=repo_root.as_posix()) + + print("Testing...") + env = os.environ.copy() + env['ICONTRACT_SLOW'] = 'true' + + subprocess.check_call([ + "coverage", "run", "--source", "persipubsub", "-m", "unittest", + "discover", "tests" + ], + cwd=repo_root.as_posix(), + env=env) + + subprocess.check_call(["coverage", "report"]) + + print("Doctesting...") + subprocess.check_call( + ["python3", "-m", "doctest", (repo_root / "README.rst").as_posix()]) + for pth in (repo_root / "persipubsub").glob("**/*.py"): + subprocess.check_call(["python3", "-m", "doctest", pth.as_posix()]) + + print("pyicontract-lint'ing...") + for pth in (repo_root / "persipubsub").glob("**/*.py"): + subprocess.check_call(["pyicontract-lint", pth.as_posix()]) + + print("Twine'ing...") + subprocess.check_call(["python3", "setup.py", "sdist", "bdist_wheel"], + cwd=repo_root.as_posix()) + subprocess.check_call(["twine", "check", "dist/*"], + cwd=repo_root.as_posix()) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py/pylint.rc b/py/pylint.rc new file mode 100644 index 0000000..7b1f5c3 --- /dev/null +++ b/py/pylint.rc @@ -0,0 +1,10 @@ +[TYPECHECK] +ignored-modules = numpy +ignored-classes = numpy,PurePath +generated-members=bottle\.request\.forms\.decode,bottle\.request\.query\.decode + +[FORMAT] +max-line-length=80 + +[MESSAGES CONTROL] +disable=too-few-public-methods,abstract-class-little-used,len-as-condition,bad-continuation,bad-whitespace, no-member, duplicate-code \ No newline at end of file diff --git a/py/requirements-doc.txt b/py/requirements-doc.txt new file mode 100644 index 0000000..eccd910 --- /dev/null +++ b/py/requirements-doc.txt @@ -0,0 +1,4 @@ +sphinx>=1.8.4,<2 +sphinx-autodoc-typehints>=1.6.0 +sphinx-icontract>=2.0.0 +sphinx-rtd-theme>=0.4.2 \ No newline at end of file diff --git a/py/setup.py b/py/setup.py new file mode 100644 index 0000000..0f916ad --- /dev/null +++ b/py/setup.py @@ -0,0 +1,69 @@ +"""A setuptools based setup module. + +See: +https://packaging.python.org/en/latest/distributing.html +https://github.com/pypa/sampleproject +""" +import os + +from setuptools import find_packages, setup + +import persipubsub_meta + +here = os.path.abspath(os.path.dirname(__file__)) # pylint: disable=invalid-name + +with open(os.path.join(here, 'README.rst'), encoding='utf-8') as fid: + long_description = fid.read().strip() # pylint: disable=invalid-name + +setup( + name=persipubsub_meta.__title__, + version=persipubsub_meta.__version__, + description=persipubsub_meta.__description__, + long_description=long_description, + url=persipubsub_meta.__url__, + author=persipubsub_meta.__author__, + author_email=persipubsub_meta.__author_email__, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + license='License :: OSI Approved :: MIT License', + keywords= + 'persistent publisher subscriber lmdb MQ message queue thread-safe process-safe', + packages=find_packages( + exclude=['tests', 'obsolete', 'messages', 'example']), + install_requires=[ + # yapf: disable + 'typing-extensions>=3.7.2', + 'icontract>=2.0.2,<3', + 'lmdb>=0.94,<1' + # yapf: enable + ], + extras_require={ + 'dev': [ + # yapf: disable + 'mypy==0.670', + 'pylint==2.2.2', + 'yapf==0.26.0', + 'tox>=3.7.0', + 'coverage>=4.5.2,<5', + 'pydocstyle>=3.0.0,<4', + 'pyicontract-lint>=2.0.0,<3', + 'docutils>=0.14,<1', + 'isort>=4.3.4,<5', + 'pygments>=2.3.1,<3', + 'twine>=1.12.1,<2', + 'setuptools>=40.8.0,<41', + 'wheel>=0.33.0,<1', + 'temppathlib>=1.0.3,<2' + # yapf: enable + ] + }, + py_modules=['persipubsub', 'persipubsub_meta'], + package_data={ + "persipubsub": ["typed"], + '.': ['LICENSE', 'README.rst'] + }) diff --git a/py/style.yapf b/py/style.yapf new file mode 100644 index 0000000..ef924b1 --- /dev/null +++ b/py/style.yapf @@ -0,0 +1,6 @@ +[style] +based_on_style = pep8 +spaces_before_comment = 2 +split_before_logical_operator = true +column_limit = 80 +coalesce_brackets = true \ No newline at end of file diff --git a/py/tests/__init__.py b/py/tests/__init__.py new file mode 100644 index 0000000..8122f5d --- /dev/null +++ b/py/tests/__init__.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Test persipubsub.""" + +import unittest + +import lmdb +import temppathlib + +import persipubsub + +# pylint: disable=missing-docstring + +LMDB_PAGE_SIZE = 4096 + +TEST_MSG_TIMEOUT = 1 # type: int +TEST_HWM_MSG_NUM = 10 # type: int +TEST_HWM_LMDB_SIZE = LMDB_PAGE_SIZE * 2 # type: int + +TEST_MAX_DB_NUM = 1024 # type: int +TEST_MAX_DB_SIZE_BYTES = TEST_HWM_LMDB_SIZE * TEST_MAX_DB_NUM # type: int +TEST_MAX_READER_NUM = 1024 # type: int + +# define all encodings here +ENCODING = 'utf-8' +BYTES_LENGTH = 8 +BYTES_ORDER = 'big' + +# define all database names here +DATA_DB = "data_db".encode(ENCODING) # msg_id | data +PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber +META_DB = "meta_db".encode(ENCODING) # msg_id | metadata +QUEUE_DB = "queue_db".encode(ENCODING) # queue_pth | all queue data + + +class TestPersiPubSub(unittest.TestCase): + def test_get_data(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = lmdb.open(path=tmp_dir.path.as_posix(), max_dbs=2) + + with env.begin(write=True) as txn: + queue_db = env.open_db(key=QUEUE_DB, txn=txn) + txn.put( + key=persipubsub.MSG_TIMEOUT_SECS_KEY, + value=persipubsub.int_to_bytes(500), + db=queue_db) + txn.put( + key=persipubsub.MAX_MSGS_NUM_KEY, + value=persipubsub.int_to_bytes(1000), + db=queue_db) + txn.put( + key=persipubsub.HWM_DB_SIZE_BYTES_KEY, + value=persipubsub.int_to_bytes(1024**3), + db=queue_db) + txn.put( + key=persipubsub.STRATEGY_KEY, + value=persipubsub.str_to_bytes("prune_last"), + db=queue_db) + txn.put( + key=persipubsub.SUBSCRIBER_IDS_KEY, + value=persipubsub.str_to_bytes(""), + db=queue_db) + + queue_data = persipubsub.lookup_queue_data(env=env) + + self.assertEqual(500, queue_data.msg_timeout_secs) + self.assertEqual(1000, queue_data.max_msgs_num) + self.assertEqual(1024**3, queue_data.hwm_db_size_bytes) + self.assertEqual("prune_last", queue_data.strategy) + self.assertEqual([], queue_data.subscriber_ids) + + +if __name__ == '__main__': + unittest.main() diff --git a/py/tests/component_publisher.py b/py/tests/component_publisher.py new file mode 100644 index 0000000..09317db --- /dev/null +++ b/py/tests/component_publisher.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +"""Publisher component for live test.""" + +import pathlib + +import persipubsub.environment + +# pylint: disable=missing-docstring + + +def send_thread(env: persipubsub.environment.Environment, num_msg: int) -> None: + pub = env.new_publisher() + + for _ in range(num_msg): + pub.send(msg="hello subscriber".encode('utf-8')) + + +def send_process(path: pathlib.Path, num_msg: int) -> None: + env = persipubsub.environment.Environment(path=path) + pub = env.new_publisher() + + for _ in range(num_msg): + pub.send(msg="hello subscriber".encode('utf-8')) diff --git a/py/tests/component_subscriber.py b/py/tests/component_subscriber.py new file mode 100644 index 0000000..5685b92 --- /dev/null +++ b/py/tests/component_subscriber.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +"""Subscriber component for live test.""" + +import pathlib +import time + +import persipubsub.environment + +# pylint: disable=missing-docstring + + +# pylint: disable=too-many-arguments +def receive_thread(path: pathlib.Path, + env: persipubsub.environment.Environment, + identifier: str, + num_msg: int, + timeout: int = 2, + retries: int = 10, + method_timeout: int = 10) -> None: + sub = env.new_subscriber(identifier=identifier) + + received_msg = 0 + + start = time.time() + while True: + if received_msg == num_msg: + break + elif time.time() - start >= method_timeout: + raise TimeoutError + else: + with sub.receive(timeout=timeout, retries=retries) as msg: + if msg is not None: + received_msg += 1 + + result = path / identifier + result.write_text('pass') + + +# pylint: disable=too-many-arguments +def receive_process(path: pathlib.Path, + identifier: str, + num_msg: int, + timeout: int = 2, + retries: int = 10, + method_timeout: int = 60) -> None: + env = persipubsub.environment.Environment(path=path) + sub = env.new_subscriber(identifier=identifier) + + received_msg = 0 + + start = time.time() + while True: + if received_msg == num_msg: + break + elif time.time() - start >= method_timeout: + raise TimeoutError + else: + with sub.receive(timeout=timeout, retries=retries) as msg: + if msg is not None: + received_msg += 1 + + result = path / identifier + result.write_text('pass') diff --git a/py/tests/test_control.py b/py/tests/test_control.py new file mode 100644 index 0000000..2a5cb50 --- /dev/null +++ b/py/tests/test_control.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +"""Test control unit.""" + +import datetime +import pathlib +import unittest +from typing import List + +import lmdb +import temppathlib + +import persipubsub.control +import persipubsub.queue +import tests + +# pylint: disable=missing-docstring +# pylint: disable=protected-access + + +def setup(path: pathlib.Path, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + control = persipubsub.control.Control(path=path) + + hwm = persipubsub.queue.HighWaterMark() + strategy = persipubsub.queue.Strategy.prune_first + + control.init(subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) + + return control + + +class TestControl(unittest.TestCase): + def test_initialize_all(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + control = setup(path=tmp_dir.path, sub_list=['sub']) + + expected_db_keys = [ + b'data_db', b'meta_db', b'pending_db', b'queue_db', b'sub' + ] + db_keys = [] # type: List[bytes] + + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin() as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + db_keys.append(key) + + self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) + + def test_del_sub(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + control = setup(path=tmp_dir.path, sub_list=["sub1", "sub2"]) + + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) + + control._remove_sub(sub_id="sub2") + + expected_db_keys = [ + b'data_db', b'meta_db', b'pending_db', b'queue_db', b'sub1' + ] + db_keys = [] # type: List[bytes] + + with control.queue.env.begin() as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + db_keys.append(key) + + self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) + + def test_clear_all_subs(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + control = setup(path=tmp_dir.path, sub_list=["sub1", "sub2"]) + + control.clear_all_subscribers() + + expected_db_keys = [ + b'data_db', b'meta_db', b'pending_db', b'queue_db' + ] + db_keys = [] # type: List[bytes] + + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin() as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + db_keys.append(key) + + self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) + + def test_prune_dangling_messages(self) -> None: + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + control = setup(path=tmp_dir.path, sub_list=["sub"]) + + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) + assert isinstance(control.queue.hwm, + persipubsub.queue.HighWaterMark) + control.queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT + + with control.queue.env.begin(write=True) as txn: + sub_db = control.queue.env.open_db( + key='sub'.encode(tests.ENCODING), txn=txn, create=False) + + txn.put(key="timeout_msg".encode(tests.ENCODING), db=sub_db) + txn.put(key="valid_msg".encode(tests.ENCODING), db=sub_db) + with control.queue.env.begin(write=True) as txn: + data_db = control.queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + txn.put( + key="popped_msg".encode(tests.ENCODING), + value="I'm data".encode(tests.ENCODING), + db=data_db) + txn.put( + key="timeout_msg".encode(tests.ENCODING), + value="I'm data too".encode(tests.ENCODING), + db=data_db) + txn.put( + key="valid_msg".encode(tests.ENCODING), + value="Free me!".encode(tests.ENCODING), + db=data_db) + with control.queue.env.begin(write=True) as txn: + pending_db = control.queue.env.open_db( + key=tests.PENDING_DB, txn=txn, create=False) + + txn.put( + key="popped_msg".encode(tests.ENCODING), + value=int(0).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=pending_db) + txn.put( + key="timeout_msg".encode(tests.ENCODING), + value=int(1).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=pending_db) + txn.put( + key="valid_msg".encode(tests.ENCODING), + value=int(1).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=pending_db) + + with control.queue.env.begin(write=True) as txn: + meta_db = control.queue.env.open_db( + key=tests.META_DB, txn=txn, create=False) + + txn.put( + key="popped_msg".encode(tests.ENCODING), + value=int(datetime.datetime.utcnow().timestamp()).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=meta_db) + txn.put( + key="timeout_msg".encode(tests.ENCODING), + value=int(datetime.datetime.utcnow().timestamp() - + tests.TEST_MSG_TIMEOUT - 5).to_bytes( + length=tests.BYTES_LENGTH, + byteorder=tests.BYTES_ORDER), + db=meta_db) + txn.put( + key="valid_msg".encode(tests.ENCODING), + value=int(datetime.datetime.utcnow().timestamp()).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=meta_db) + + control.prune_dangling_messages() + + dbs = [sub_db, data_db, pending_db, meta_db] + + expected_remaining_entries = 4 + remaining_entries = 0 + + # pylint: disable=invalid-name + for db in dbs: + with control.queue.env.begin(db=db) as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + remaining_entries += 1 + self.assertEqual(b'valid_msg', key) + + self.assertEqual(expected_remaining_entries, remaining_entries) + + def test_prune_all_messages_for_subscriber(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + control = setup(path=tmp_dir.path, sub_list=["sub"]) + + msg = persipubsub.str_to_bytes("hello world!") + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) + control.queue.put(msg=msg) + control.queue.put(msg=msg) + + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key=persipubsub.str_to_bytes('sub'), txn=txn, create=False) + sub_stat = txn.stat(db=sub_db) + self.assertEqual(2, sub_stat['entries']) + + control._prune_all_messages_for(sub_id="sub") + + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key=persipubsub.str_to_bytes('sub'), txn=txn, create=False) + sub_stat = txn.stat(db=sub_db) + self.assertEqual(0, sub_stat['entries']) + + def test_is_initialized(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + control = setup(path=tmp_dir.path, sub_list=["sub"]) + + self.assertTrue(control.is_initialized()) + + def test_is_not_initialized(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + control = persipubsub.control.Control(path=tmp_dir.path) + self.assertFalse(control.is_initialized()) + + +if __name__ == '__main__': + unittest.main() diff --git a/py/tests/test_environment.py b/py/tests/test_environment.py new file mode 100644 index 0000000..aa487d3 --- /dev/null +++ b/py/tests/test_environment.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +"""Test environment.""" + +import unittest + +import temppathlib + +import persipubsub.control +import persipubsub.environment +import persipubsub.publisher +import persipubsub.queue +import persipubsub.subscriber + +# pylint: disable=missing-docstring +# pylint: disable=protected-access + + +class TestEnvironment(unittest.TestCase): + def test_new_environment(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + self.assertIsInstance(env, persipubsub.environment.Environment) + + def test_new_control(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + ctl = env.new_control() + self.assertIsInstance(ctl, persipubsub.control.Control) + + def test_new_publisher(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + ctl = env.new_control() + ctl.init() + pub = env.new_publisher() + self.assertIsInstance(pub, persipubsub.publisher.Publisher) + + def test_new_subscriber(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + ctl = env.new_control() + ctl.init() + sub = env.new_subscriber(identifier="sub") + self.assertIsInstance(sub, persipubsub.subscriber.Subscriber) + + +if __name__ == '__main__': + unittest.main() diff --git a/py/tests/test_live.py b/py/tests/test_live.py new file mode 100644 index 0000000..e697658 --- /dev/null +++ b/py/tests/test_live.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python +"""Test persipubsub live.""" +import multiprocessing +import pathlib +import threading +import time +import unittest + +import lmdb +import temppathlib + +import persipubsub.environment +import persipubsub.publisher +import persipubsub.queue +import persipubsub.subscriber +import tests +import tests.component_publisher +import tests.component_subscriber + +# pylint: disable=missing-docstring + +FIRST_SUB_STARTED_READING = False + + +def subscriber_receive_first(sub: persipubsub.subscriber.Subscriber) -> None: + with sub.receive() as _: + global FIRST_SUB_STARTED_READING # pylint: disable=global-statement + FIRST_SUB_STARTED_READING = True + time.sleep(2) + + +def subscriber_receive_second(sub: persipubsub.subscriber.Subscriber) -> None: + global FIRST_SUB_STARTED_READING # pylint: disable=global-statement + + start = time.time() + while not FIRST_SUB_STARTED_READING: + if time.time() - start > 10: + raise TimeoutError + time.sleep(0.1) + + with sub.receive() as _: + if time.time() - start < 2: + assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub.queue.env, lmdb.Environment) + assert isinstance(sub.queue.env.path(), str) + assert isinstance(sub.identifier, str) + result = pathlib.Path(sub.queue.env.path()) / sub.identifier + result.write_text('pass') + else: + assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub.queue.env, lmdb.Environment) + assert isinstance(sub.queue.env.path(), str) + assert isinstance(sub.identifier, str) + result = pathlib.Path(sub.queue.env.path()) / sub.identifier + result.write_text('fail') + + +def receive(sub: persipubsub.subscriber.Subscriber, + num_msg: int, + timeout: int = 2, + method_timeout: int = 60) -> None: + received_msg = 0 + + start = time.time() + while True: + if received_msg == num_msg: + break + elif time.time() - start >= method_timeout: + raise TimeoutError + + with sub.receive(timeout=timeout) as msg: + if msg is not None: + received_msg += 1 + + assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub.queue.env, lmdb.Environment) + assert isinstance(sub.queue.env.path(), str) + assert isinstance(sub.identifier, str) + result = pathlib.Path(sub.queue.env.path()) / sub.identifier + result.write_text('pass') + + +def send(pub: persipubsub.publisher.Publisher, num_msg: int) -> None: + for _ in range(num_msg): + pub.send(msg="hello subscriber".encode('utf-8')) + + +class TestLive(unittest.TestCase): + def test_multithreaded_communication_one_publisher_one_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub']) + + result = tmp_dir.path / "sub" + result.touch() + + pub = env.new_publisher() + sub = env.new_subscriber(identifier='sub') + + num_msg = 1000 + pub_thread = threading.Thread( + target=send, kwargs={ + 'pub': pub, + 'num_msg': num_msg + }) + sub_thread = threading.Thread( + target=receive, kwargs={ + 'sub': sub, + 'num_msg': num_msg + }) + + pub_thread.start() + sub_thread.start() + + for thread in [pub_thread, sub_thread]: + thread.join() + + self.assertEqual('pass', result.read_text()) + + # pylint: disable=too-many-locals + def test_multithreaded_communication_two_publisher_two_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub1', 'sub2']) + + result1 = tmp_dir.path / "sub1" + result1.touch() + result2 = tmp_dir.path / "sub2" + result2.touch() + + pub1 = env.new_publisher() + pub2 = env.new_publisher() + sub1 = env.new_subscriber(identifier='sub1') + sub2 = env.new_subscriber(identifier='sub2') + + num_msg = 1000 + + pub1_thread = threading.Thread( + target=send, kwargs={ + 'pub': pub1, + 'num_msg': num_msg + }) + sub1_thread = threading.Thread( + target=receive, kwargs={ + 'sub': sub1, + 'num_msg': 2 * num_msg + }) + pub2_thread = threading.Thread( + target=send, kwargs={ + 'pub': pub2, + 'num_msg': num_msg + }) + sub2_thread = threading.Thread( + target=receive, kwargs={ + 'sub': sub2, + 'num_msg': 2 * num_msg + }) + + pub1_thread.start() + sub1_thread.start() + pub2_thread.start() + sub2_thread.start() + + for thread in [pub1_thread, sub1_thread, pub2_thread, sub2_thread]: + thread.join() + + self.assertEqual('pass', result1.read_text()) + self.assertEqual('pass', result2.read_text()) + + def test_multithreaded_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub']) + + result = tmp_dir.path / "sub" + result.touch() + + num_msg = 1000 + pub_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + sub_thread = threading.Thread( + target=tests.component_subscriber.receive_thread, + kwargs={ + 'path': tmp_dir.path, + 'env': env, + 'identifier': 'sub', + 'num_msg': num_msg + }) + pub_thread.start() + sub_thread.start() + + for thread in [pub_thread, sub_thread]: + thread.join() + + self.assertEqual('pass', result.read_text()) + + def test_multithreaded_two_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub1', 'sub2']) + + result1 = tmp_dir.path / "sub1" + result1.touch() + result2 = tmp_dir.path / "sub2" + result2.touch() + + num_msg = 300 + pub1_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + sub1_thread = threading.Thread( + target=tests.component_subscriber.receive_thread, + kwargs={ + 'path': tmp_dir.path, + 'env': env, + 'identifier': 'sub1', + 'num_msg': 2 * num_msg, + 'method_timeout': 60 + }) + pub2_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + sub2_thread = threading.Thread( + target=tests.component_subscriber.receive_thread, + kwargs={ + 'path': tmp_dir.path, + 'env': env, + 'identifier': 'sub2', + 'num_msg': 2 * num_msg, + 'method_timeout': 60 + }) + pub1_thread.start() + sub1_thread.start() + pub2_thread.start() + sub2_thread.start() + + for thread in [pub1_thread, sub1_thread, pub2_thread, sub2_thread]: + thread.join() + + self.assertEqual('pass', result1.read_text()) + self.assertEqual('pass', result2.read_text()) + + def test_multiprocess_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub']) + env.env.close() + + result = tmp_dir.path / "sub" + result.touch() + + num_msg = 1000 + pub_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + sub_process = multiprocessing.Process( + target=tests.component_subscriber.receive_process, + kwargs={ + 'path': tmp_dir.path, + 'identifier': 'sub', + 'num_msg': num_msg + }) + pub_process.start() + sub_process.start() + + for process in [pub_process, sub_process]: + process.join() + + self.assertEqual('pass', result.read_text()) + + def test_multiprocess_two_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub1', 'sub2']) + + result1 = tmp_dir.path / "sub1" + result1.touch() + result2 = tmp_dir.path / "sub2" + result2.touch() + + num_msg = 300 + pub1_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + sub1_process = multiprocessing.Process( + target=tests.component_subscriber.receive_process, + kwargs={ + 'path': tmp_dir.path, + 'identifier': 'sub1', + 'num_msg': 2 * num_msg, + 'method_timeout': 60 + }) + pub2_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + sub2_process = multiprocessing.Process( + target=tests.component_subscriber.receive_process, + kwargs={ + 'path': tmp_dir.path, + 'identifier': 'sub2', + 'num_msg': 2 * num_msg, + 'method_timeout': 60 + }) + pub1_process.start() + sub1_process.start() + pub2_process.start() + sub2_process.start() + + for process in [ + pub1_process, sub1_process, pub2_process, sub2_process + ]: + process.join() + + self.assertEqual('pass', result1.read_text()) + self.assertEqual('pass', result2.read_text()) + + def test_multithreaded_race_condition_of_the_component_publisher( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + control = env.new_control(['sub']) + + num_msg = 50 + num_threads = 50 + + threads = [] + for _ in range(num_threads): + pub_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + + pub_thread.start() + threads.append(pub_thread) + + for thread in threads: + thread.join() + + assert isinstance(control.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key='sub'.encode('utf-8'), txn=txn) + self.assertEqual(num_msg * num_threads, + txn.stat(db=sub_db)['entries']) + + def test_multiprocess_race_condition_of_the_component_publisher( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + control = env.new_control(['sub']) + + num_msg = 50 + num_processes = 50 + + processes = [] + for _ in range(num_processes): + pub_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + + pub_process.start() + processes.append(pub_process) + + for process in processes: + process.join() + + assert isinstance(control.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key='sub'.encode('utf-8'), txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=sub_db)['entries']) + + def test_multiprocess_race_condition_of_the_component_publisher_one_env( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + control = env.new_control(subscriber_ids=['sub']) + + num_msg = 50 + num_processes = 50 + + processes = [] + for _ in range(num_processes): + pub_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'num_msg': num_msg, + 'path': tmp_dir.path, + }) + + pub_process.start() + processes.append(pub_process) + + for process in processes: + process.join() + + assert isinstance(control.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key='sub'.encode('utf-8'), txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=sub_db)['entries']) + data_db = control.queue.env.open_db( + key=persipubsub.DATA_DB, txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=data_db)['entries']) + meta_db = control.queue.env.open_db( + key=persipubsub.META_DB, txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=meta_db)['entries']) + pending_db = control.queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=pending_db)['entries']) + + def test_2_subscriber_non_blocking(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(subscriber_ids=['sub1', 'sub2']) + + pub = env.new_publisher() + sub1 = env.new_subscriber(identifier='sub1') + sub2 = env.new_subscriber(identifier='sub2') + + pub.send(msg='msg for two subscriber'.encode('utf-8')) + + sub1_thread = threading.Thread( + target=subscriber_receive_first, kwargs={ + 'sub': sub1, + }) + sub2_thread = threading.Thread( + target=subscriber_receive_second, kwargs={ + 'sub': sub2, + }) + sub1_thread.start() + sub2_thread.start() + + for thread in [sub1_thread, sub2_thread]: + thread.join() + + assert isinstance(sub2.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub2.queue.env, lmdb.Environment) + assert isinstance(sub2.queue.env.path(), str) + assert isinstance(sub2.identifier, str) + result = pathlib.Path(sub2.queue.env.path()) / sub2.identifier + self.assertEqual('pass', result.read_text()) + + +if __name__ == '__main__': + unittest.main() diff --git a/py/tests/test_publisher.py b/py/tests/test_publisher.py new file mode 100644 index 0000000..19ee2a6 --- /dev/null +++ b/py/tests/test_publisher.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +"""Test publisher.""" + +import unittest +from typing import List + +import lmdb +import temppathlib + +import persipubsub.control +import persipubsub.environment +import persipubsub.queue +import tests + +# pylint: disable=missing-docstring +# pylint: disable=protected-access + + +def setup(env: persipubsub.environment.Environment, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + hwm = persipubsub.queue.HighWaterMark() + strategy = persipubsub.queue.Strategy.prune_first + + control = env.new_control( + subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) + + return control + + +class TestPublisher(unittest.TestCase): + def test_send(self) -> None: + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + _ = setup(env=env, sub_list=['sub']) + + pub = env.new_publisher() + + msg = "Hello world!".encode(tests.ENCODING) + pub.send(msg=msg) + + subscriber = "sub".encode(tests.ENCODING) + + with env.env.begin(write=False) as txn: + self.assertIsNotNone(txn.get(key=subscriber)) + sub_db = env.env.open_db(key=subscriber, txn=txn, create=False) + cursor = txn.cursor(db=sub_db) + self.assertTrue(cursor.first()) + + key = cursor.key() + + data_db = env.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + item = txn.get(key=key, db=data_db) + self.assertIsNotNone(item) + self.assertEqual(msg, item) + + def test_send_many(self) -> None: + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + subscriber = "sub" + _ = setup(env=env, sub_list=[subscriber]) + + pub = env.new_publisher() + + msg = "I'm a message".encode(tests.ENCODING) + msgs = [] + msg_num = 10 + for _ in range(msg_num): + msgs.append(msg) + + pub.send_many(msgs=msgs) + + assert isinstance(pub.queue, persipubsub.queue._Queue) + assert isinstance(pub.queue.env, lmdb.Environment) + with pub.queue.env.begin(write=False) as txn: + self.assertIsNotNone( + txn.get(key=subscriber.encode(tests.ENCODING))) + + sub_db = pub.queue.env.open_db( + key=subscriber.encode(tests.ENCODING), + txn=txn, + create=False) + + sub_stat = txn.stat(db=sub_db) + self.assertEqual(msg_num, sub_stat['entries']) + + data_db = pub.queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + data_stat = txn.stat(db=data_db) + self.assertEqual(msg_num, data_stat['entries']) + + +if __name__ == '__main__': + unittest.main() diff --git a/py/tests/test_queue.py b/py/tests/test_queue.py new file mode 100644 index 0000000..b49a605 --- /dev/null +++ b/py/tests/test_queue.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Test database.""" + +import time +import unittest +from typing import List + +import lmdb +import temppathlib + +import persipubsub.control +import persipubsub.environment +import persipubsub.queue +import tests + +# pylint: disable=missing-docstring +# pylint: disable=protected-access + + +def setup(env: persipubsub.environment.Environment, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + hwm = persipubsub.queue.HighWaterMark() + strategy = persipubsub.queue.Strategy.prune_first + + control = env.new_control( + subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) + + return control + + +class TestQueue(unittest.TestCase): + def test_initialize_environment(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.queue._initialize_environment( + queue_dir=tmp_dir.path) + + self.assertDictEqual({ + 'branch_pages': 0, + 'entries': 0, + 'depth': 0, + 'leaf_pages': 0, + 'overflow_pages': 0, + 'psize': 4096 + }, env.stat()) + self.assertDictEqual({ + 'last_txnid': 0, + 'last_pgno': 1, + 'map_size': 32 * 1024**3, + 'map_addr': 0, + 'max_readers': 1024, + 'num_readers': 1 + }, env.info()) + + def test_put_to_single_subscriber(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = "I'm a message.".encode(tests.ENCODING) + + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + queue.put(msg=msg) + + assert isinstance(queue.env, lmdb.Environment) + with queue.env.begin() as txn: + self.assertIsNotNone( + txn.get(key=subscriber.encode(tests.ENCODING))) + + sub_db = queue.env.open_db( + key=subscriber.encode(tests.ENCODING), + txn=txn, + create=False) + cursor = txn.cursor(db=sub_db) + self.assertTrue(cursor.first()) + key = cursor.key() + + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + value = txn.get(key=key, db=data_db) + self.assertIsNotNone(value) + self.assertEqual(msg, value) + + def test_put_multiple_subscriber(self) -> None: + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = "I'm a message.".encode(tests.ENCODING) + + sub_list = ["sub", "another_sub"] + + env = persipubsub.environment.Environment(path=tmp_dir.path) + _ = setup(env=env, sub_list=sub_list) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + queue.put(msg=msg) + + assert isinstance(queue.env, lmdb.Environment) + with queue.env.begin() as txn: + self.assertIsNotNone( + txn.get(key=sub_list[0].encode(tests.ENCODING))) + + sub_db_0 = queue.env.open_db( + key=sub_list[0].encode(tests.ENCODING), + txn=txn, + create=False) + cursor = txn.cursor(db=sub_db_0) + self.assertTrue(cursor.first()) + key_0 = cursor.key() + + self.assertIsNotNone( + txn.get(key=sub_list[1].encode(tests.ENCODING))) + + sub_db_1 = queue.env.open_db( + key=sub_list[1].encode(tests.ENCODING), + txn=txn, + create=False) + cursor = txn.cursor(db=sub_db_1) + self.assertTrue(cursor.first()) + key_1 = cursor.key() + + self.assertEqual(key_0, key_1) + + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + value = txn.get(key=key_1, db=data_db) + self.assertIsNotNone(value) + self.assertEqual(msg, value) + + def test_put_many(self) -> None: + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + msg = "I'm a message".encode(tests.ENCODING) + msgs = [] + msg_num = 10 + for _ in range(msg_num): + msgs.append(msg) + + queue.put_many_flush_once(msgs=msgs) + + assert isinstance(queue.env, lmdb.Environment) + with queue.env.begin(write=False) as txn: + self.assertIsNotNone( + txn.get(key=subscriber.encode(tests.ENCODING))) + + sub_db = queue.env.open_db( + key=subscriber.encode(tests.ENCODING), + txn=txn, + create=False) + + sub_stat = txn.stat(db=sub_db) + self.assertEqual(msg_num, sub_stat['entries']) + + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + data_stat = txn.stat(db=data_db) + self.assertEqual(msg_num, data_stat['entries']) + + def test_front(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = "I'm a message.".encode(tests.ENCODING) + + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + queue.put(msg=msg) + + # pylint: disable=assignment-from-none + # pylint: disable=assignment-from-no-return + received_msg = queue.front(identifier=subscriber) + self.assertIsNotNone(received_msg) + self.assertEqual(msg, received_msg) + + def test_pop(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = "I'm a message.".encode(tests.ENCODING) + + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + queue.put(msg=msg) + + # pylint: disable=assignment-from-none + # pylint: disable=assignment-from-no-return + received_msg = queue.front(identifier=subscriber) + self.assertIsNotNone(received_msg) + + assert isinstance(queue.env, lmdb.Environment) + with queue.env.begin() as txn: + pending_db = queue.env.open_db( + key=tests.PENDING_DB, txn=txn, create=False) + + cursor = txn.cursor(db=pending_db) + self.assertTrue(cursor.first()) + pending_before_pop = cursor.value() + + queue.pop(identifier=subscriber) + + received_msg = queue.front(identifier=subscriber) + self.assertIsNone(received_msg) + + with queue.env.begin() as txn: + pending_db = queue.env.open_db( + key=tests.PENDING_DB, txn=txn, create=False) + cursor = txn.cursor(db=pending_db) + self.assertTrue(cursor.first()) + pending_after_pop = cursor.value() + + self.assertEqual( + int.from_bytes(pending_before_pop, tests.BYTES_ORDER) - 1, + int.from_bytes(pending_after_pop, tests.BYTES_ORDER)) + + def test_pop_queue_empty(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + self.assertRaises(RuntimeError, queue.pop, identifier=subscriber) + + def test_queue_initialisation(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + control = setup(env=env, sub_list=[subscriber]) + + hwm = persipubsub.queue.HighWaterMark( + msg_timeout_secs=tests.TEST_MSG_TIMEOUT, + max_msgs_num=tests.TEST_HWM_MSG_NUM, + hwm_lmdb_size_bytes=tests.TEST_HWM_LMDB_SIZE) + + control.set_hwm(hwm=hwm) + control.set_strategy( + strategy=persipubsub.queue.Strategy.prune_first) + queue = env.new_control().queue + assert isinstance(queue, persipubsub.queue._Queue) + + self.assertIsNotNone(queue.env) + assert isinstance(queue.env, lmdb.Environment) + self.assertEqual(tmp_dir.path.as_posix(), queue.env.path()) + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) + self.assertEqual(tests.TEST_HWM_LMDB_SIZE, + queue.hwm.hwm_lmdb_size_bytes) + self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.hwm.max_msgs_num) + self.assertEqual(tests.TEST_MSG_TIMEOUT, queue.hwm.msg_timeout_secs) + assert isinstance(queue.strategy, persipubsub.queue.Strategy) + self.assertEqual(persipubsub.queue.Strategy.prune_first.name, + queue.strategy.name) + self.assertEqual(['sub'], queue.subscriber_ids) + + def test_overflow_msgs_limit(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + + msg = "hello world".encode(tests.ENCODING) + + self.assertEqual(0, queue.count_msgs()) + for _ in range(tests.TEST_HWM_MSG_NUM): + queue.put(msg=msg) + + self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.count_msgs()) + + queue.put(msg=msg) + + self.assertEqual( + int(tests.TEST_HWM_MSG_NUM - int(tests.TEST_HWM_MSG_NUM / 2)), + queue.count_msgs()) + + def test_overflow_limit_size(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) + queue.hwm.hwm_lmdb_size_bytes = tests.TEST_HWM_LMDB_SIZE + + msg = ("a" * (int(tests.LMDB_PAGE_SIZE / 4))).encode(tests.ENCODING) + + while queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE: + queue.put(msg=msg) + + self.assertTrue( + queue.check_current_lmdb_size() > tests.TEST_HWM_LMDB_SIZE) + + queue.put(msg=msg) + + self.assertTrue( + queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE) + + def test_timeout(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) + queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT + + msg = "hello world".encode(tests.ENCODING) + + queue.put(msg=msg) + self.assertEqual(1, queue.count_msgs()) + queue.put(msg=msg) + self.assertEqual(2, queue.count_msgs()) + time.sleep(tests.TEST_MSG_TIMEOUT + 1) + queue.put(msg=msg) + self.assertEqual(1, queue.count_msgs()) + + def test_strategy_prune_first(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + queue.strategy = persipubsub.queue.Strategy.prune_first + + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + + for index in range(tests.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode(tests.ENCODING) + queue.put(msg=msg) + + self.assertEqual("secret message 0".encode(tests.ENCODING), + queue.front(identifier='sub')) + + msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( + tests.ENCODING) + queue.put(msg=msg) + + self.assertEqual( + "secret message {}".format( + int((tests.TEST_HWM_MSG_NUM / 2) + 1)).encode( + tests.ENCODING), queue.front(identifier='sub')) + + def test_strategy_prune_last(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + queue.strategy = persipubsub.queue.Strategy.prune_last + + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + + for index in range(tests.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode(tests.ENCODING) + queue.put(msg=msg) + + self.assertEqual("secret message 0".encode(tests.ENCODING), + queue.front(identifier='sub')) + + msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( + tests.ENCODING) + queue.put(msg=msg) + + self.assertEqual("secret message 0".encode(tests.ENCODING), + queue.front(identifier='sub')) + + +if __name__ == '__main__': + unittest.main() diff --git a/py/tests/test_subscriber.py b/py/tests/test_subscriber.py new file mode 100644 index 0000000..b0144e4 --- /dev/null +++ b/py/tests/test_subscriber.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +"""Test subscriber.""" + +import unittest +from typing import List + +import temppathlib + +import persipubsub.control +import persipubsub.environment +import persipubsub.queue +import persipubsub.subscriber +import tests + +# pylint: disable=missing-docstring +# pylint: disable=protected-access + + +def setup(env: persipubsub.environment.Environment, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + hwm = persipubsub.queue.HighWaterMark() + strategy = persipubsub.queue.Strategy.prune_first + + control = env.new_control( + subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) + + return control + + +class TestSubscriber(unittest.TestCase): + def test_receive_message(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + msg = "Hello World!".encode(tests.ENCODING) + queue.put(msg=msg) + + with sub.receive(timeout=1) as received_msg: + self.assertIsNotNone(received_msg) + self.assertEqual(msg, received_msg) + + def test_timeout_subscriber(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + with sub.receive(timeout=1) as received_msg: + self.assertIsNone(received_msg) + msg = "message send after timeout and will not be popped" \ + "".encode(tests.ENCODING) + queue.put(msg=msg) + self.assertIsNone(received_msg) + + self.assertEqual( + "message send after timeout and will not be popped".encode( + tests.ENCODING), queue.front(identifier=subscriber)) + + with sub.receive(timeout=1) as received_msg: + self.assertIsNotNone(received_msg) + self.assertEqual( + "message send after timeout and will not be popped".encode( + tests.ENCODING), received_msg) + + self.assertIsNone(queue.front(identifier=subscriber)) + + def test_pop(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + msg1 = "I'm a message".encode(tests.ENCODING) + queue.put(msg=msg1) + + msg2 = "I'm a message too".encode(tests.ENCODING) + queue.put(msg=msg2) + + sub._pop() + + with sub.receive() as msg: + self.assertIsNotNone(msg) + self.assertEqual(msg2, msg) + + def test_pop_when_empty(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + sub = env.new_subscriber(identifier=subscriber) + + self.assertRaises(RuntimeError, sub._pop) + + def test_receive_to_top(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) + + msg1 = "I'm a message".encode(tests.ENCODING) + queue.put(msg=msg1) + + msg2 = "I'm a message too".encode(tests.ENCODING) + queue.put(msg=msg2) + + with sub.receive_to_top() as msg: + self.assertIsNotNone(msg) + self.assertEqual(msg2, msg) + + +if __name__ == '__main__': + unittest.main() diff --git a/py/tox.ini b/py/tox.ini new file mode 100644 index 0000000..f660911 --- /dev/null +++ b/py/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py35,py36 + +[testenv] +deps = .[dev] +changedir = {envtmpdir} +commands = + python3 {toxinidir}/precommit.py + +setenv = + COVERAGE_FILE={envbindir}/.coverage \ No newline at end of file