From 9e10eaa6807225c7c7678c1ec3a8b8a646c4013b Mon Sep 17 00:00:00 2001 From: David Erb Date: Tue, 23 May 2023 16:43:03 +0000 Subject: [PATCH 01/18] relaxes isolation modes --- .dae-devops/Makefile | 4 +- .dae-devops/docs/conventions.rst | 4 +- .dae-devops/docs/developing.rst | 5 +- .dae-devops/docs/devops.rst | 15 +- .dae-devops/docs/docs_structure.rst | 4 +- .dae-devops/docs/installing.rst | 7 +- .dae-devops/docs/testing.rst | 12 +- .dae-devops/prepare_git_dependencies.sh | 4 +- .dae-devops/project.yaml | 3 +- .devcontainer/Dockerfile | 4 +- .devcontainer/devcontainer.json | 4 +- .github/CONTRIBUTING.rst | 4 +- .../actions/install_requirements/action.yml | 4 +- .github/dependabot.yml | 4 +- .github/pages/index.html | 4 +- .github/pages/make_switcher.py | 4 +- .github/workflows/code.yml | 4 +- .github/workflows/docs.yml | 4 +- .github/workflows/docs_clean.yml | 4 +- .github/workflows/linkcheck.yml | 4 +- .gitlab-ci.yml | 4 +- docs/_static/css/custom.css | 4 +- docs/conf.py | 4 +- pyproject.toml | 6 +- src/dls_normsql/aiomysql.py | 613 ++++++++++++++++++ src/dls_normsql/aiosqlite.py | 58 +- src/dls_normsql/constants.py | 1 + src/dls_normsql/databases.py | 5 + tests/conftest.py | 5 - tests/my_table_definition.py | 2 +- tests/test_aiomysql.py | 51 ++ tests/test_database.py | 106 ++- 32 files changed, 841 insertions(+), 120 deletions(-) create mode 100644 src/dls_normsql/aiomysql.py create mode 100644 tests/test_aiomysql.py diff --git a/.dae-devops/Makefile b/.dae-devops/Makefile index 1b7fcfc..43aff4f 100644 --- a/.dae-devops/Makefile +++ b/.dae-devops/Makefile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql # --------------------------------------------------------------------- @@ -69,4 +69,4 @@ publish_docs: cp -r build/html/* $(DOCS_PUBLISH_ROOT) -# dae_devops_fingerprint 3f13d455c444ad39930aabfc68547c2c +# dae_devops_fingerprint 7ad8e9ac2116461b84a0200de66ec86f diff --git a/.dae-devops/docs/conventions.rst b/.dae-devops/docs/conventions.rst index 31323fc..daa5ac9 100644 --- a/.dae-devops/docs/conventions.rst +++ b/.dae-devops/docs/conventions.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-normsql Naming conventions @@ -31,4 +31,4 @@ repository lowercase, hyphens -.. # dae_devops_fingerprint 81e582297295ea8d1145aeb4fb60ffef +.. # dae_devops_fingerprint d57378dce1a17dd03d4629d169efd5ca diff --git a/.dae-devops/docs/developing.rst b/.dae-devops/docs/developing.rst index 1c78388..5e60b4b 100644 --- a/.dae-devops/docs/developing.rst +++ b/.dae-devops/docs/developing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-normsql Developing @@ -22,7 +22,6 @@ Make sure to have at least python version 3.9 then:: Install the package in edit mode which will also install all its dependencies:: $ cd dls-normsql - $ export PIP_FIND_LINKS=/dls_sw/apps/bxflow/artifacts $ pip install -e .[dev] Now you may begin modifying the code. @@ -36,4 +35,4 @@ If you plan to modify the docs, you will need to:: -.. # dae_devops_fingerprint 550d6069746935ee4e08d958dd046fa2 +.. # dae_devops_fingerprint 06258b784a6611f9c5473dd7ab11d8c1 diff --git a/.dae-devops/docs/devops.rst b/.dae-devops/docs/devops.rst index 35648e5..fd67200 100644 --- a/.dae-devops/docs/devops.rst +++ b/.dae-devops/docs/devops.rst @@ -1,23 +1,24 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-normsql Devops ======================================================================= -There exists a a configuration file called ``.dae-devops/project.yaml``. +In the top level of the repository there exists a configuration file called ``.dae-devops/project.yaml``. This file defines the project information needed for CI/CD. It is parsed by the ``dae_devops.force`` command which creates these files: - pyproject.toml +- .githib/* - .gitlab-ci.yml - .dae-devops/Makefile - .dae-devops/docs/* Local CI/CD execution ------------------------------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ All the CI/CD ops which are run by the git server can be run at the command line. @@ -32,15 +33,15 @@ Validation of the code:: $ make -f .dae-devops/Makefile validate_pytest $ make -f .dae-devops/Makefile validate_docs -Packaging:: +Packaging (for the Diamond intranet):: $ make -f .dae-devops/Makefile package_pip -Publishing:: +Publishing (for the Diamond intranet):: $ make -f .dae-devops/Makefile publish_pip $ make -f .dae-devops/Makefile publish_docs +The Diamond intranet commands are not used for production. The production packaging and publishing are handled in the GitHub Actions workflows mechanism. - -.. # dae_devops_fingerprint 35ea37a143cb4560425194afc2bad8ed +.. # dae_devops_fingerprint 315edec766c8c72d01b0fa9707eddd83 diff --git a/.dae-devops/docs/docs_structure.rst b/.dae-devops/docs/docs_structure.rst index a4abbb9..513c3e5 100644 --- a/.dae-devops/docs/docs_structure.rst +++ b/.dae-devops/docs/docs_structure.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-normsql About the documentation @@ -21,4 +21,4 @@ improve most documentation - often immensely. `More information on this topic. `_ -.. # dae_devops_fingerprint 68095839584c6e3455c1bd06350312f4 +.. # dae_devops_fingerprint ec4b9c8af9a0c16a88c8afd2f10d8044 diff --git a/.dae-devops/docs/installing.rst b/.dae-devops/docs/installing.rst index 9440602..3a8530d 100644 --- a/.dae-devops/docs/installing.rst +++ b/.dae-devops/docs/installing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-normsql Installing @@ -26,7 +26,6 @@ installation will not interfere with any existing Python software:: You can now use ``pip`` to install the library and its dependencies:: - $ export PIP_FIND_LINKS=/dls_sw/apps/bxflow/artifacts $ python3 -m pip install dls-normsql If you require a feature that is not currently released you can also install @@ -34,10 +33,10 @@ from git:: $ python3 -m pip install git+https://gitlab.diamond.ac.uk/kbp43231/dls-normsql.git -The library should now be installed and the commandline interface on your path. +The library should now be installed and the commandline should be available. You can check the version that has been installed by typing:: $ dls-normsql --version $ dls-normsql --version-json -.. # dae_devops_fingerprint f0f3fa3bcf53a551add4ee39538eb44a +.. # dae_devops_fingerprint 9c071b85e47a709fdf94b44c012d3f03 diff --git a/.dae-devops/docs/testing.rst b/.dae-devops/docs/testing.rst index 844b411..45f0b09 100644 --- a/.dae-devops/docs/testing.rst +++ b/.dae-devops/docs/testing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-normsql Testing @@ -11,17 +11,13 @@ If you want to run the tests, first get a copy of the code per the instructions Then you can run all tests by:: - $ pytest - -Or this, which is the command used by the CI runner. - - $ make -f .dae-devops/Makefile validate_pytest + $ tox -q -e pytest To run a single test you can do:: $ pytest tests/the_test_you_want.py -If you want to see more output of the test while it's running you can do: +If you want to see more output of the test while it's running you can do:: $ pytest -sv -ra --tb=line tests/the_test_you_want.py @@ -35,4 +31,4 @@ This allows peeking in there to see what's been written by the test. -.. # dae_devops_fingerprint 16abfa1d4f582c07e1ea9281771b8f4f +.. # dae_devops_fingerprint 16749d7794075d5edc250b7e3a37ae35 diff --git a/.dae-devops/prepare_git_dependencies.sh b/.dae-devops/prepare_git_dependencies.sh index d284eff..ae27539 100644 --- a/.dae-devops/prepare_git_dependencies.sh +++ b/.dae-devops/prepare_git_dependencies.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql me=${BASH_SOURCE} @@ -13,4 +13,4 @@ function __install { } -# dae_devops_fingerprint d2a7e27cfad2e9f2547dc2b0da58fdd1 \ No newline at end of file +# dae_devops_fingerprint 1a34e24b1d5a45cb1ca98a9cbad72c6e \ No newline at end of file diff --git a/.dae-devops/project.yaml b/.dae-devops/project.yaml index 654a0a1..ec76a4b 100644 --- a/.dae-devops/project.yaml +++ b/.dae-devops/project.yaml @@ -13,5 +13,6 @@ primary: dependencies: - type: pypi list: - - aiosqlite - dls-utilpack + - aiosqlite + - aiomysql diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e8d31bb..bd945be 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql # This file is for use as a devcontainer and a runtime container @@ -40,4 +40,4 @@ ENV PATH=/venv/bin:$PATH ENTRYPOINT ["dls-normsql"] CMD ["--version"] -# dae_devops_fingerprint 41ae7527130d2727aa06cd91ff8e5702 +# dae_devops_fingerprint 6dc80a2a370a712769cdb9563fa17fc0 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d9ed733..16e1f3c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ // ********** Please don't edit this file! -// ********** It has been generated automatically by dae_devops version 0.5.2. +// ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. // ********** For repository_name dls-normsql // For format details, see https://containers.dev/implementors/json_reference/ @@ -57,4 +57,4 @@ "postCreateCommand": "pip install -e .[dev]" } -// dae_devops_fingerprint bf6676a0739eeb0e671f3518d7739877 +// dae_devops_fingerprint 554efbeb9af2bdb211b7e057e098d375 diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 86d834c..77701e9 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-normsql Contributing to the project @@ -38,4 +38,4 @@ should follow. .. _Developer Guide: https://diamondlightsource.github.io/dls-normsql/main/developer/how-to/contribute.html -.. # dae_devops_fingerprint e4fc983b36dd78a7c7fefcb6ac9fdd13 +.. # dae_devops_fingerprint fbbfd64dec105165b3aa9295ebd7cd6e diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 4349378..b71b4f1 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql name: Install requirements @@ -61,4 +61,4 @@ runs: shell: bash -# dae_devops_fingerprint 8d7ad9a9e4bbb3b504abc3a0f33579ee +# dae_devops_fingerprint 60c68e2129decf28ed4e20b91c02763e diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b0af985..90d39b7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql # To get started with Dependabot version updates, you'll need to specify which @@ -19,4 +19,4 @@ updates: schedule: interval: "weekly" -# dae_devops_fingerprint b8759630d6502121150bac83a60ac05d +# dae_devops_fingerprint c70363631c8e5cfa8a7053e5419dd1d4 diff --git a/.github/pages/index.html b/.github/pages/index.html index 83d3f23..62121ad 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -1,5 +1,5 @@ - + @@ -14,4 +14,4 @@ - + diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index 692a7cc..172c0d6 100644 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql import json @@ -102,4 +102,4 @@ def main(args=None): if __name__ == "__main__": main() -# dae_devops_fingerprint 321882bf975a37cedc9e7def02e120fa +# dae_devops_fingerprint 82ea8a19d01c89a49e187624838f2126 diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 59473b9..63c9248 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql name: Code CI @@ -212,4 +212,4 @@ jobs: with: password: ${{ secrets.PYPI_TOKEN }} -# dae_devops_fingerprint 0eca35d77828ef5ba3c51d0d68628f12 +# dae_devops_fingerprint 39b6ade54ad4279a47cb3da654b3721e diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b18390b..5109be2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql name: Docs CI @@ -56,4 +56,4 @@ jobs: publish_dir: .github/pages keep_files: true -# dae_devops_fingerprint ea6935f4eb0060233fc35dcccd2a0a36 +# dae_devops_fingerprint fc42ff4e21df624b69d71d1daf733ff7 diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index 1ce2b98..7f92cde 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql name: Docs Cleanup CI @@ -46,4 +46,4 @@ jobs: git commit -am "Removing redundant docs version $DOCS_VERSION" git push -# dae_devops_fingerprint 53d885e9033e0abb28bcc63d8252ca03 +# dae_devops_fingerprint 062f4bc18f25b11b80cbca161780c289 diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index ca2194b..52c7a06 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql name: Link Check @@ -27,4 +27,4 @@ jobs: - name: Check links run: tox -e docs build -- -b linkcheck -# dae_devops_fingerprint f31df44c662aa0a356bb2a62fe64eade +# dae_devops_fingerprint 364f1f117764b2f547d91993c76ca046 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69dc1bb..d5e9228 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql variables: @@ -87,4 +87,4 @@ package_pip: # # The validate_docs artifacts are in the build/html folder. # - make -f .dae-devops/Makefile publish_docs -# dae_devops_fingerprint 19ac89e7287f70ae7f5f96292102fde8 +# dae_devops_fingerprint b29bc99027c830817f5b2dd3c6c69c43 diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 818c05b..ac95ef8 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,5 +1,5 @@ /* ********** Please don't edit this file! */ -/* ********** It has been generated automatically by dae_devops version 0.5.2. */ +/* ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. */ /* ********** For repository_name dls-normsql */ /* The theme normally has this, but I think it's ok to use the full width of the window in all @media sizes. @@ -15,4 +15,4 @@ max-width: 100%; } -/* dae_devops_fingerprint b13b064702739049faa6ce9119be9039 */ +/* dae_devops_fingerprint 6d64e77ebbc876fc118c61804d4caaba */ diff --git a/docs/conf.py b/docs/conf.py index b89d646..78cca0b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql from pathlib import Path @@ -197,4 +197,4 @@ def setup(app): app.connect("source-read", ultimateReplace) -# dae_devops_fingerprint 404e6b95750b57333b0799fcd1a2c408 +# dae_devops_fingerprint a5475bf5aabb8f8db72e43bae4457f5e diff --git a/pyproject.toml b/pyproject.toml index a993abb..4033cf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-normsql [build-system] @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] description = "Normalized API over various sql libraries." -dependencies = ["aiosqlite", "dls-utilpack"] +dependencies = ["dls-utilpack", "aiosqlite", "aiomysql"] dynamic = ["version"] license.file = "LICENSE" readme = "README.rst" @@ -102,4 +102,4 @@ source = ["src", "**/site-packages/"] [tool.tox] legacy_tox_ini = "[tox]\nskipsdist=True\n\n[testenv:{pre-commit,mypy,pytest,docs}]\n# Don't create a virtualenv for the command, requires tox-direct plugin\ndirect = True\npassenv = *\nallowlist_externals = \n pytest \n pre-commit\n mypy\n sphinx-build\n sphinx-autobuild\ncommands =\n pytest: pytest {posargs}\n mypy: mypy src tests {posargs}\n pre-commit: pre-commit run --all-files {posargs}\n docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html\n" -# dae_devops_fingerprint a8f7c8e6af4793c950a1b15a2e697dcb +# dae_devops_fingerprint ba05247624b5791448c05976d419d99d diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py new file mode 100644 index 0000000..6fbaa33 --- /dev/null +++ b/src/dls_normsql/aiomysql.py @@ -0,0 +1,613 @@ +import asyncio +import glob + +# This class produces log entries. +import logging +import os +import shutil +import warnings +from collections import OrderedDict +from datetime import datetime +from typing import Any, List, Optional + +import aiomysql + +# Utilities. +from dls_utilpack.callsign import callsign +from dls_utilpack.explain import explain +from dls_utilpack.isodatetime import isodatetime_filename +from dls_utilpack.require import require + +from dls_normsql.constants import CommonFieldnames, RevisionFieldnames, Tablenames +from dls_normsql.table_definition import TableDefinition + +logger = logging.getLogger(__name__) + +connect_lock = asyncio.Lock() +apply_revisions_lock = asyncio.Lock() + +# ---------------------------------------------------------------------------------------- +class Aiomysql: + """ + Class with coroutines for creating and querying a sqlite database. + """ + + # ---------------------------------------------------------------------------------------- + def __init__(self, specification): + """ + Construct object. Do not connect to database. + """ + + s = f"{callsign(self)} specification" + + t = require(s, specification, "type_specific_tbd") + + self.__host = require(s, t, "host") + + self.__port = require(s, t, "port") + + self.__username = require(s, t, "username") + + self.__password = require(s, t, "password") + + self.__database_name = require(s, t, "database_name") + + self.__connection = None + + self.__tables = {} + + # Deriving class has not established its latest revision? + if not hasattr(self, "LATEST_REVISION") or self.LATEST_REVISION is None: + # Presume it is 1. + self.LATEST_REVISION = 1 + + self.__backup_restore_lock = asyncio.Lock() + + # Last undo position. + self.__last_restore = 0 + + # ---------------------------------------------------------------------------------------- + def __parameterize(self, sql: str, subs: List[Any]): + """ + Connect to database at filename given in constructor. + """ + + sql = sql.replace("?", "%s") + + return sql + + # ---------------------------------------------------------------------------------------- + async def connect(self, should_drop_database=False): + """ + Connect to database at filename given in constructor. + """ + + async with connect_lock: + should_create_schemas = False + + logger.debug(f"connecting to mysql") + + self.__connection = await aiomysql.connect( + host=self.__host, + port=self.__port, + user=self.__username, + password=self.__password, + ) + + # Set the transaction isolation level so that commits on one connection are available to other connections quickly. + # The default was REPEATABLE READ, which is stricter, but I found I could not get committed data immediately. + # TODO: Consider implications of setting aiomysql to TRANSACTION ISOLATION LEVEL READ COMMITTED. + await self.execute("SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED") + + try: + await self.execute(f"USE {self.__database_name}") + database_exists = True + except aiomysql.OperationalError: + database_exists = False + + if database_exists and should_drop_database: + await self.execute(f"DROP DATABASE {self.__database_name}") + database_exists = False + + if not database_exists: + await self.execute(f"CREATE DATABASE {self.__database_name}") + await self.execute(f"USE {self.__database_name}") + should_create_schemas = True + + # Commit the statement to apply the isolation level + await self.__connection.commit() + + # Execute the SQL query to get the isolation level + records = await self.query("SELECT @@session.transaction_isolation") + + # Fetch the result + isolation_level = records[0] + logger.debug(f"isolation_level is {isolation_level}") + + # Let the base class contribute its table definitions to the in-memory list. + await self.add_table_definitions() + + if should_create_schemas: + await self.create_schemas() + await self.insert( + Tablenames.REVISION, [{"number": self.LATEST_REVISION}] + ) + + # Emit the name of the database file for positive confirmation on console. + logger.info( + f"{callsign(self)} database name is {self.__database_name} code revision {self.LATEST_REVISION}" + ) + + # ---------------------------------------------------------------------------------------- + async def apply_revisions(self): + """ + Apply revision updates to databse if needed. + """ + + # TODO: Consider how to lock database while running applying_revisions. + # TODO: Establish transaction arouund apply_revisions with rollback if error. + async with apply_revisions_lock: + try: + records = await self.query( + f"SELECT number FROM {Tablenames.REVISION}", + why="get database revision", + ) + if len(records) == 0: + old_revision = 0 + else: + old_revision = records[0]["number"] + except Exception as exception: + logger.warning( + f"could not get revision, presuming legacy database with no table: {exception}" + ) + old_revision = 0 + + if old_revision < self.LATEST_REVISION: + # Backup before applying revisions. + logger.debug( + f"[BKREVL] backing up before updating from revision {old_revision} to revision {self.LATEST_REVISION}" + ) + + await self.backup() + + for revision in range(old_revision, self.LATEST_REVISION): + logger.debug(f"updating to revision {revision+1}") + await self.apply_revision(revision + 1) + await self.update( + Tablenames.REVISION, + {"number": self.LATEST_REVISION}, + "1 = 1", + why="update database revision", + ) + else: + logger.debug( + f"[BKREVL] no need to update persistent revision {old_revision}" + f" which matches code revision {self.LATEST_REVISION}" + ) + + # ---------------------------------------------------------------------------------------- + async def apply_revision(self, revision): + logger.debug(f"updating to revision {revision}") + # Updating to revision 1 presumably means + # this is a legacy database with no revision table in it. + if revision == 1: + logger.info(f"creating {Tablenames.REVISION} table") + await self.create_table(Tablenames.REVISION) + await self.insert(Tablenames.REVISION, [{"revision": revision}]) + + # ---------------------------------------------------------------------------------------- + async def disconnect(self): + + if self.__connection is not None: + logger.debug(f"[DISSHU] {callsign(self)} disconnecting") + self.__connection.close() + self.__connection = None + + # ---------------------------------------------------------------------------------------- + async def __create_directory(self, filename): + + directory, filename = os.path.split(filename) + + if not os.path.exists(directory): + # Make sure that parent directories which get created will have public permission. + umask = os.umask(0) + os.umask(umask & ~0o0777) + os.makedirs(directory) + os.umask(umask) + + # ---------------------------------------------------------------------------------------- + def add_table_definition(self, table_definition): + + self.__tables[table_definition.name] = table_definition + + # ---------------------------------------------------------------------------------------- + async def add_table_definitions(self): + + self.add_table_definition(RevisionTableDefinition(self)) + + # ---------------------------------------------------------------------------------------- + async def begin(self): + """ + Begin transaction. + """ + + logger.debug("beginning transaction") + + await self.__connection.begin() + + # ---------------------------------------------------------------------------------------- + async def commit(self): + """ + Commit transaction. + """ + + logger.debug("committing transaction") + + await self.__connection.commit() + + # ---------------------------------------------------------------------------------------- + async def rollback(self): + """ + Roll back transaction. + """ + logger.debug("rolling back transaction") + + await self.__connection.rollback() + + # ---------------------------------------------------------------------------------------- + async def create_schemas(self): + + for table in self.__tables.values(): + await self.create_table(table) + + # ---------------------------------------------------------------------------------------- + async def create_table(self, table): + """ + Wipe and re-create the table in the database. + """ + + # If table is a string, presume it's a table name. + if isinstance(table, str): + table = require("table definitions", self.__tables, table) + + async with self.__connection.cursor() as cursor: + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + await cursor.execute("DROP TABLE IF EXISTS %s" % (table.name)) + + fields_sql = [] + indices_sql = [] + + for field_name in table.fields: + field = table.fields[field_name] + field_type = field["type"].upper() + if field_type == "TEXT PRIMARY KEY": + field_type = "VARCHAR(64) PRIMARY KEY" + fields_sql.append("%s %s" % (field_name, field_type)) + if field.get("index", False): + if field_type == "TEXT": + index_length = "(128)" + else: + index_length = "" + indices_sql.append( + "CREATE INDEX %s_%s ON %s(%s%s)" + % (table.name, field_name, table.name, field_name, index_length) + ) + + sql = "CREATE TABLE %s\n(%s)" % (table.name, ",\n ".join(fields_sql)) + + logger.debug("\n%s\n%s" % (sql, "\n".join(indices_sql))) + + await cursor.execute(sql) + + for sql in indices_sql: + await cursor.execute(sql) + + # ---------------------------------------------------------------------------------------- + async def insert( + self, + table, + rows, + why=None, + ) -> None: + """ + Insert one or more rows. + Each row is a dictionary. + The first row is expected to define the keys for all rows inserted. + Keys in the rows are ignored if not defined in the table schema. + Table schema columns not specified in the first row's keys will get their sql-defined default values. + """ + + if len(rows) == 0: + return + + # If table is a string, presume it's a table name. + if isinstance(table, str): + table = require("table definitions", self.__tables, table) + + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + + values_rows = [] + + insertable_fields = [] + for field in table.fields: + # The first row is expected to define the keys for all rows inserted. + if field in rows[0]: + insertable_fields.append(field) + elif field == CommonFieldnames.CREATED_ON: + insertable_fields.append(field) + + qmarks = ["?"] * len(insertable_fields) + + for row in rows: + values_row = [] + for field in table.fields: + if field == CommonFieldnames.CREATED_ON: + created_on = row.get(field) + if created_on is None: + created_on = now + values_row.append(created_on) + elif field in row: + values_row.append(row[field]) + values_rows.append(values_row) + + sql = "INSERT INTO %s\n (%s)\n VALUES (%s)" % ( + table.name, + ", ".join(insertable_fields), + ", ".join(qmarks), + ) + + sql = self.__parameterize(sql, values_rows) + + try: + async with self.__connection.cursor() as cursor: + await cursor.executemany(sql, values_rows) + + if why is None: + logger.debug("\n%s\n%s" % (sql, values_rows)) + else: + logger.debug("%s:\n%s\n%s" % (why, sql, values_rows)) + + except (TypeError, aiomysql.OperationalError) as e: + if why is None: + raise RuntimeError(f"failed to execute {sql}") + else: + raise RuntimeError(f"failed to execute {why}: {sql}") + + # ---------------------------------------------------------------------------------------- + async def update( + self, + table, + row, + where, + subs=None, + why=None, + ): + """ + Update specified fields to all rows matching selection. + """ + + # If table is a string, presume it's a table name. + if isinstance(table, str): + table = require("table definitions", self.__tables, table) + + values_row = [] + qmarks = [] + + for field in table.fields: + if field == CommonFieldnames.UUID or field == CommonFieldnames.AUTOID: + continue + if field not in row: + continue + qmarks.append("%s = ?" % (field)) + values_row.append(row[field]) + + if len(values_row) == 0: + raise RuntimeError("no fields in record match database table") + + sql = "UPDATE %s SET\n %s\nWHERE %s" % ( + table.name, + ",\n ".join(qmarks), + where, + ) + + if subs is not None: + values_row.extend(subs) + + sql = self.__parameterize(sql, subs) + + async with self.__connection.cursor() as cursor: + try: + await cursor.execute(sql, values_row) + rowcount = cursor.rowcount + + if why is None: + logger.debug( + "%d rows from:\n%s\nvalues %s" % (rowcount, sql, values_row) + ) + else: + logger.debug( + "%d rows from %s:\n%s\nvalues %s" + % (rowcount, why, sql, values_row) + ) + + except (TypeError, aiomysql.OperationalError): + if why is None: + raise RuntimeError(f"failed to execute {sql}") + else: + raise RuntimeError(f"failed to execute {why}: {sql}") + + return rowcount + + # ---------------------------------------------------------------------------------------- + async def execute( + self, + sql, + subs=None, + why=None, + ): + """ + Execute a sql statement. + If subs is a list of lists, then these are presumed the values for executemany. + """ + + async with self.__connection.cursor() as cursor: + try: + sql = self.__parameterize(sql, subs) + + # Subs is a list of lists? + if ( + isinstance(subs, list) + and len(subs) > 0 + and isinstance(subs[0], list) + ): + logger.debug(f"inserting {len(subs)} of {len(subs[0])}") + await cursor.executemany(sql, subs) + else: + await cursor.execute(sql, subs) + + if why is None: + if cursor.rowcount > 0: + logger.debug( + f"{cursor.rowcount} records affected by\n{sql}\nvalues {subs}" + ) + else: + logger.debug(f"{sql}\nvalues {subs}") + else: + if cursor.rowcount > 0: + logger.debug( + f"{cursor.rowcount} records affected by {why}:\n{sql} values {subs}" + ) + else: + logger.debug(f"{why}: {sql}\nvalues {subs}") + except (TypeError, aiomysql.OperationalError): + if why is None: + raise RuntimeError(f"failed to execute {sql}") + else: + raise RuntimeError(f"failed to execute {why}: {sql}") + + # ---------------------------------------------------------------------------------------- + async def query(self, sql, subs=None, why=None): + + if subs is None: + subs = {} + + sql = self.__parameterize(sql, subs) + + cursor = None + async with self.__connection.cursor() as cursor: + try: + cursor = await self.__connection.cursor() + await cursor.execute(sql, subs) + rows = await cursor.fetchall() + cols = [] + for col in cursor.description: + cols.append(col[0]) + + if why is None: + logger.debug("%d records from: %s" % (len(rows), sql)) + else: + logger.debug("%d records from %s: %s" % (len(rows), why, sql)) + records = [] + for row in rows: + record = OrderedDict() + for index, col in enumerate(cols): + record[col] = row[index] + records.append(record) + return records + except (TypeError, aiomysql.OperationalError) as exception: + if why is None: + raise RuntimeError(explain(exception, f"executing {sql}")) + else: + raise RuntimeError(explain(exception, f"executing {why}: {sql}")) + + # ---------------------------------------------------------------------------------------- + async def backup(self): + """ + Back up database to timestamped location. + """ + + async with self.__backup_restore_lock: + # Prune all the restores which were orphaned. + directory = self.__backup_directory + if directory is None: + raise RuntimeError("no backup directory supplied in confirmation") + + basename, suffix = os.path.splitext(os.path.basename(self.__database_name)) + + filenames = glob.glob(f"{directory}/{basename}.*{suffix}") + + filenames.sort(reverse=True) + + for restore in range(self.__last_restore): + logger.debug( + f"[BACKPRU] removing {restore}-th restore {filenames[restore]}" + ) + os.remove(filenames[restore]) + + self.__last_restore = 0 + + timestamp = isodatetime_filename() + to_filename = f"{directory}/{basename}.{timestamp}{suffix}" + + await self.disconnect() + try: + await self.__create_directory(to_filename) + shutil.copy2(self.__database_name, to_filename) + logger.debug(f"backed up to {to_filename}") + except Exception: + raise RuntimeError( + f"copy {self.__database_name} to {to_filename} failed" + ) + finally: + await self.connect() + + # ---------------------------------------------------------------------------------------- + async def restore(self, nth): + """ + Restore database from timestamped location. + """ + + async with self.__backup_restore_lock: + directory = self.__backup_directory + if directory is None: + raise RuntimeError("no backup directory supplied in confirmation") + + basename, suffix = os.path.splitext(os.path.basename(self.__database_name)) + + filenames = glob.glob(f"{directory}/{basename}.*{suffix}") + + filenames.sort(reverse=True) + + if nth >= len(filenames): + raise RuntimeError( + f"restoration index {nth} is more than available {len(filenames)}" + ) + + from_filename = filenames[nth] + + await self.disconnect() + try: + shutil.copy2(from_filename, self.__database_name) + logger.debug( + f"restored nth {nth} out of {len(filenames)} from {from_filename}" + ) + except Exception: + raise RuntimeError( + f"copy {from_filename} to {self.__database_name} failed" + ) + finally: + await self.connect() + + self.__last_restore = nth + + +# ---------------------------------------------------------------------------------------- +class RevisionTableDefinition(TableDefinition): + # ---------------------------------------------------------------------------------------- + def __init__(self, database): + TableDefinition.__init__(self, "revision") + + self.fields[RevisionFieldnames.CREATED_ON] = {"type": "TEXT", "index": True} + self.fields[RevisionFieldnames.NUMBER] = {"type": "INTEGER", "index": False} diff --git a/src/dls_normsql/aiosqlite.py b/src/dls_normsql/aiosqlite.py index c24b41b..b0b5d76 100644 --- a/src/dls_normsql/aiosqlite.py +++ b/src/dls_normsql/aiosqlite.py @@ -72,7 +72,7 @@ def __init__(self, specification): self.__last_restore = 0 # ---------------------------------------------------------------------------------------- - async def connect(self): + async def connect(self, should_drop_database=False): """ Connect to database at filename given in constructor. """ @@ -92,6 +92,11 @@ async def connect(self): self.__connection = await aiosqlite.connect(self.__filename) self.__connection.row_factory = aiosqlite.Row + # Set isolation level such that all statements not in an explicit transaction are autocommitted immediately and visible on other connections. + # TODO: Consider the ramifications of setting aiosqlite isolation_level to None. + self.__connection.isolation_level = None + logger.debug(f"isolation_level is {self.__connection.isolation_level}") + # rows = await self.query("PRAGMA journal_mode", why="query journal mode") # logger.debug(f"journal mode rows {json.dumps(rows)}") @@ -208,35 +213,44 @@ async def add_table_definitions(self): self.add_table_definition(RevisionTableDefinition(self)) + # ---------------------------------------------------------------------------------------- + async def begin(self): + """ + Begin transaction. + """ + + await self.__connection.execute("BEGIN") + # ---------------------------------------------------------------------------------------- async def commit(self): """ - Commit transaction, if any outstanding. + Commit transaction. """ - if self.__connection.in_transaction: - await self.__connection.commit() + await self.__connection.commit() # ---------------------------------------------------------------------------------------- async def rollback(self): """ - Roll back transaction, if any outstanding. + Roll back transaction. """ - if self.__connection.in_transaction: - await self.__connection.rollback() + await self.__connection.rollback() # ---------------------------------------------------------------------------------------- - async def create_schemas(self, should_commit: Optional[bool] = True): + async def create_schemas(self): - for table in self.__tables.values(): - await self.create_table(table, should_commit=False) + await self.begin() - if should_commit: - await self.__connection.commit() + try: + for table in self.__tables.values(): + await self.create_table(table) + await self.commit() + except Exception: + await self.rollback() # ---------------------------------------------------------------------------------------- - async def create_table(self, table, should_commit: Optional[bool] = True): + async def create_table(self, table): """ Wipe and re-create the table in the database. """ @@ -268,16 +282,12 @@ async def create_table(self, table, should_commit: Optional[bool] = True): for sql in indices_sql: await self.__connection.execute(sql) - if should_commit: - await self.__connection.commit() - # ---------------------------------------------------------------------------------------- async def insert( self, table, rows, why=None, - should_commit: Optional[bool] = True, ) -> None: """ Insert one or more rows. @@ -334,9 +344,6 @@ async def insert( else: logger.debug("%s:\n%s\n%s" % (why, sql, values_rows)) - if should_commit: - await self.__connection.commit() - except aiosqlite.OperationalError: if why is None: raise RuntimeError(f"failed to execute {sql}") @@ -351,7 +358,6 @@ async def update( where, subs=None, why=None, - should_commit: Optional[bool] = True, ): """ Update specified fields to all rows matching selection. @@ -388,9 +394,6 @@ async def update( cursor = await self.__connection.execute(sql, values_row) rowcount = cursor.rowcount - if should_commit: - await self.__connection.commit() - if why is None: logger.debug( "%d rows from:\n%s\nvalues %s" % (rowcount, sql, values_row) @@ -409,9 +412,7 @@ async def update( return rowcount # ---------------------------------------------------------------------------------------- - async def execute( - self, sql, subs=None, why=None, should_commit: Optional[bool] = True - ): + async def execute(self, sql, subs=None, why=None): """ Execute a sql statement. If subs is a list of lists, then these are presumed the values for executemany. @@ -426,9 +427,6 @@ async def execute( else: cursor = await self.__connection.execute(sql, subs) - if should_commit: - await self.__connection.commit() - if why is None: if cursor.rowcount > 0: logger.debug( diff --git a/src/dls_normsql/constants.py b/src/dls_normsql/constants.py index c98e7b9..ed6cf69 100644 --- a/src/dls_normsql/constants.py +++ b/src/dls_normsql/constants.py @@ -1,6 +1,7 @@ # ---------------------------------------------------------------------------------------- class ClassTypes: AIOSQLITE = "dls_normsql.aiosqlite" + AIOMYSQL = "dls_normsql.aiomysql" # ---------------------------------------------------------------------------------------- diff --git a/src/dls_normsql/databases.py b/src/dls_normsql/databases.py index 8773a77..68acd21 100644 --- a/src/dls_normsql/databases.py +++ b/src/dls_normsql/databases.py @@ -43,4 +43,9 @@ def lookup_class(self, class_type): return Aiosqlite + if class_type == ClassTypes.AIOMYSQL: + from dls_normsql.aiomysql import Aiomysql + + return Aiomysql + raise NotFound("unable to get database class for type %s" % (class_type)) diff --git a/tests/conftest.py b/tests/conftest.py index ed9be81..941bae3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,12 +34,7 @@ def logging_setup(): # Turn off noisy debug. logging.getLogger("asyncio").setLevel("WARNING") - logging.getLogger("pika").setLevel("WARNING") - logging.getLogger("stomp").setLevel("WARNING") - logging.getLogger("luigi-interface").setLevel("WARNING") - logging.getLogger("luigi.bx_scheduler").setLevel("INFO") logging.getLogger("urllib3.connectionpool").setLevel("INFO") - logging.getLogger("dls_utilpack.things").setLevel("INFO") # Cover the version. diff --git a/tests/my_table_definition.py b/tests/my_table_definition.py index cac6ef5..cba349d 100644 --- a/tests/my_table_definition.py +++ b/tests/my_table_definition.py @@ -20,5 +20,5 @@ def __init__(self): self.fields[CommonFieldnames.CREATED_ON] = {"type": "TEXT", "index": True} self.fields["my_field"] = {"type": "TEXT", "index": True} - for i in range(200): + for i in range(10): self.fields["my_other_field_%03d" % (i)] = {"type": "TEXT", "index": True} diff --git a/tests/test_aiomysql.py b/tests/test_aiomysql.py new file mode 100644 index 0000000..a9d7ac3 --- /dev/null +++ b/tests/test_aiomysql.py @@ -0,0 +1,51 @@ +import logging + +import aiomysql + +from tests.base_tester import BaseTester + +logger = logging.getLogger(__name__) + + +# ---------------------------------------------------------------------------------------- +class TestAiomysql: + def test(self, logging_setup, output_directory): + """ + Tests the sqlite implementation of Database. + """ + + # Database specification. + database_specification = {} + + # Test direct SQL access to the database. + AiomysqlTester().main( + database_specification, + output_directory, + ) + + +# ---------------------------------------------------------------------------------------- +class AiomysqlTester(BaseTester): + """ + Test direct SQL access to the database. + """ + + async def _main_coroutine(self, database_specification, output_directory): + """ """ + pool = await aiomysql.create_pool( + host="xchem-mysql", + port=3306, + user="root", + password="root", + db="mysql", + ) + + async with pool.acquire() as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT 42;") + logger.debug(cursor.description) + (r,) = await cursor.fetchone() + assert r == 42 + + pool.close() + await pool.wait_closed() diff --git a/tests/test_database.py b/tests/test_database.py index f178de9..d113bfa 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -9,7 +9,7 @@ # ---------------------------------------------------------------------------------------- -class TestDatabase: +class TestDatabaseAiosqlite: def test(self, logging_setup, output_directory): """ Tests the sqlite implementation of Database. @@ -28,6 +28,32 @@ def test(self, logging_setup, output_directory): ) +# ---------------------------------------------------------------------------------------- +class TestDatabaseAiomysql: + def test(self, logging_setup, output_directory): + """ + Tests the mysql implementation of Database. + """ + + # Database specification. + database_specification = { + "type": ClassTypes.AIOMYSQL, + "type_specific_tbd": { + "database_name": "dls_normsql_pytest", + "host": "xchem-mysql", + "port": 3306, + "username": "root", + "password": "root", + }, + } + + # Test direct SQL access to the database. + DatabaseTester().main( + database_specification, + output_directory, + ) + + # ---------------------------------------------------------------------------------------- class DatabaseTester(BaseTester): """ @@ -38,15 +64,20 @@ async def _main_coroutine(self, database_specification, output_directory): """ """ databases = Databases() - database = databases.build_object(database_specification) - database.add_table_definition(MyTableDefinition()) + database1 = databases.build_object(database_specification) + database2 = databases.build_object(database_specification) + + database1.add_table_definition(MyTableDefinition()) try: # Connect to database. - await database.connect() + await database1.connect(should_drop_database=True) + + # Connect to second database. + await database2.connect() # Write one record. - await database.insert( + await database1.insert( "my_table", [ { @@ -62,17 +93,17 @@ async def _main_coroutine(self, database_specification, output_directory): # Query all the records. all_sql = "SELECT * FROM my_table" - records = await database.query(all_sql) + records = await database1.query(all_sql) assert len(records) == 2 - # Bulk insert more records. + # Bulk insert more records to test multiple substitutions. insertable_records = [ ["f1", "{'a': 'f111'}"], ["f2", "{'a': 'f112'}"], ["f3", "{'a': 'f113'}"], ["f4", "{'a': 'f114'}"], ] - await database.execute( + await database1.execute( f"INSERT INTO my_table" f" ({CommonFieldnames.UUID}, my_field)" " VALUES (?, ?)", @@ -80,37 +111,68 @@ async def _main_coroutine(self, database_specification, output_directory): ) # ------------------------------------------------------------ + await database1.begin() + # Single insert another record. - insertable_record = ["z1", "{'z': 'z111'}"] + insertable_record = ["y1", "{'y': 'y111'}"] - await database.execute( + await database1.execute( f"INSERT INTO my_table" f" ({CommonFieldnames.UUID}, my_field)" " VALUES (?, ?)", insertable_record, - should_commit=False, ) # Roll it back. - await database.rollback() + await database1.rollback() # Query all the records. all_sql = "SELECT * FROM my_table" - records = await database.query(all_sql) + records = await database1.query(all_sql) assert len(records) == 6 # ------------------------------------------------------------ - # Disonnect from the database. - await database.disconnect() + await database1.begin() - # Reconnect to database. - await database.connect() + # Single insert another record. + insertable_record = ["z1", "{'z': 'z111'}"] - # Query all the records. - all_sql = "SELECT * FROM my_table" - records = await database.query(all_sql) + await database1.execute( + f"INSERT INTO my_table" + f" ({CommonFieldnames.UUID}, my_field)" + " VALUES (?, ?)", + insertable_record, + ) + + # Query all the records from second database, should not see the uncommitted insert. + records = await database2.query(all_sql) assert len(records) == 6 + # Commit it. + await database1.commit() + + # Query all the records. + records = await database1.query(all_sql) + assert len(records) == 7 + + # Query all the records from second database, should now see the committed insert. + records = await database2.query(all_sql) + assert len(records) == 7 + + # ------------------------------------------------------------ + # Disconnect from the databases. + await database1.disconnect() + + logger.debug("------------------------------------------") + + # Reconnect on the same database object. + await database1.connect() + + # Query all the records. + records = await database1.query(all_sql) + assert len(records) == 7 + finally: - # Connect from the database... necessary to allow asyncio loop to exit. - await database.disconnect() + # Disonnect from the databases... necessary to allow asyncio loop to exit. + await database2.disconnect() + await database1.disconnect() From 4eca9520b2d346a160c9167f7c68e4f9948a9cab Mon Sep 17 00:00:00 2001 From: David Erb Date: Tue, 23 May 2023 16:57:12 +0000 Subject: [PATCH 02/18] defaults to relaxed isolation modes --- src/dls_normsql/aiomysql.py | 26 ++++++++++++-------------- src/dls_normsql/aiosqlite.py | 15 ++++++++++++--- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index 6fbaa33..c3eea38 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -42,6 +42,9 @@ def __init__(self, specification): t = require(s, specification, "type_specific_tbd") + # We might use values in type_specific_tbd during the connection method. + self.__type_specific_tbd = t + self.__host = require(s, t, "host") self.__port = require(s, t, "port") @@ -94,10 +97,15 @@ async def connect(self, should_drop_database=False): password=self.__password, ) - # Set the transaction isolation level so that commits on one connection are available to other connections quickly. - # The default was REPEATABLE READ, which is stricter, but I found I could not get committed data immediately. - # TODO: Consider implications of setting aiomysql to TRANSACTION ISOLATION LEVEL READ COMMITTED. - await self.execute("SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED") + # Allow the possibility to set the transaction isolation level so that commits on one connection are available to other connections quickly. + # The default was REPEATABLE READ, which is stricter, but I found I could not get committed data immediately at a second connection. + # TODO: Consider implications of setting aiomysql to the more relaxed TRANSACTION ISOLATION LEVEL READ COMMITTED. + isolation_level = self.__type_specific_tbd.get( + "isolation_level", "READ COMMITTED" + ) + await self.execute( + f"SET GLOBAL TRANSACTION ISOLATION LEVEL {isolation_level}" + ) try: await self.execute(f"USE {self.__database_name}") @@ -114,16 +122,6 @@ async def connect(self, should_drop_database=False): await self.execute(f"USE {self.__database_name}") should_create_schemas = True - # Commit the statement to apply the isolation level - await self.__connection.commit() - - # Execute the SQL query to get the isolation level - records = await self.query("SELECT @@session.transaction_isolation") - - # Fetch the result - isolation_level = records[0] - logger.debug(f"isolation_level is {isolation_level}") - # Let the base class contribute its table definitions to the in-memory list. await self.add_table_definitions() diff --git a/src/dls_normsql/aiosqlite.py b/src/dls_normsql/aiosqlite.py index b0b5d76..ef48128 100644 --- a/src/dls_normsql/aiosqlite.py +++ b/src/dls_normsql/aiosqlite.py @@ -48,6 +48,9 @@ def __init__(self, specification): f"{callsign(self)} specification", specification, "filename" ) + # Default is an empty type_specific_tbd. + self.__type_specific_tbd = specification.get("type_specific_tbd", {}) + # Backup directory default is the path where the filename is. self.__backup_directory = specification.get( "backup_directory", os.path.dirname(self.__filename) @@ -92,10 +95,16 @@ async def connect(self, should_drop_database=False): self.__connection = await aiosqlite.connect(self.__filename) self.__connection.row_factory = aiosqlite.Row - # Set isolation level such that all statements not in an explicit transaction are autocommitted immediately and visible on other connections. + # Set isolation level such that all statements which are not already in an explicit transaction + # are autocommitted immediately and visible on other connections. + # This might be less efficient? But it's nice when monitoring a sqlite file with a management tool. # TODO: Consider the ramifications of setting aiosqlite isolation_level to None. - self.__connection.isolation_level = None - logger.debug(f"isolation_level is {self.__connection.isolation_level}") + # Possible values are None, '' (default), DEFERRED, and EXCLUSIVE. + isolation_level = self.__type_specific_tbd.get("isolation_level", None) + self.__connection.isolation_level = isolation_level + logger.debug( + f"isolation_level is now set to '{self.__connection.isolation_level}'" + ) # rows = await self.query("PRAGMA journal_mode", why="query journal mode") # logger.debug(f"journal mode rows {json.dumps(rows)}") From 88b6b81e363c9b2b9b01694cd24bb5da4bd7e0e4 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 04:44:14 +0000 Subject: [PATCH 03/18] adds required constructor arg for database definition --- .dae-devops/project.yaml | 2 ++ src/dls_normsql/aiomysql.py | 6 +++++- src/dls_normsql/aiosqlite.py | 10 +++++++++- src/dls_normsql/databases.py | 4 ++-- tests/my_database_definition.py | 35 +++++++++++++++++++++++++++++++++ tests/test_database.py | 17 ++++++++++------ 6 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 tests/my_database_definition.py diff --git a/.dae-devops/project.yaml b/.dae-devops/project.yaml index ec76a4b..62545ea 100644 --- a/.dae-devops/project.yaml +++ b/.dae-devops/project.yaml @@ -16,3 +16,5 @@ primary: - dls-utilpack - aiosqlite - aiomysql + # cryptography package is required for aiomysql sha256_password or caching_sha2_password auth methods + - cryptography diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index c3eea38..c8ba410 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -33,10 +33,11 @@ class Aiomysql: """ # ---------------------------------------------------------------------------------------- - def __init__(self, specification): + def __init__(self, specification, database_definition_object): """ Construct object. Do not connect to database. """ + self.__database_definition_object = database_definition_object s = f"{callsign(self)} specification" @@ -223,6 +224,9 @@ async def add_table_definitions(self): self.add_table_definition(RevisionTableDefinition(self)) + # Let the database definition object do its thing. + await self.__database_definition_object.add_table_definitions(self) + # ---------------------------------------------------------------------------------------- async def begin(self): """ diff --git a/src/dls_normsql/aiosqlite.py b/src/dls_normsql/aiosqlite.py index ef48128..1665596 100644 --- a/src/dls_normsql/aiosqlite.py +++ b/src/dls_normsql/aiosqlite.py @@ -40,7 +40,7 @@ class Aiosqlite: """ # ---------------------------------------------------------------------------------------- - def __init__(self, specification): + def __init__(self, specification, database_definition_object): """ Construct object. Do not connect to database. """ @@ -62,6 +62,8 @@ def __init__(self, specification): self.__connection = None + self.__database_definition_object = database_definition_object + self.__tables = {} # Deriving class has not established its latest revision? @@ -192,6 +194,9 @@ async def apply_revision(self, revision): await self.create_table(Tablenames.REVISION) await self.insert(Tablenames.REVISION, [{"revision": revision}]) + # Let the database definition object do its thing. + self.__database_definition_object.apply_revisions(self) + # ---------------------------------------------------------------------------------------- async def disconnect(self): @@ -222,6 +227,9 @@ async def add_table_definitions(self): self.add_table_definition(RevisionTableDefinition(self)) + # Let the database definition object do its thing. + await self.__database_definition_object.add_table_definitions(self) + # ---------------------------------------------------------------------------------------- async def begin(self): """ diff --git a/src/dls_normsql/databases.py b/src/dls_normsql/databases.py index 68acd21..4ecb056 100644 --- a/src/dls_normsql/databases.py +++ b/src/dls_normsql/databases.py @@ -19,14 +19,14 @@ class Databases: """ # ---------------------------------------------------------------------------------------- - def build_object(self, specification): + def build_object(self, specification, database_definition_object): """""" class_type = require("database specification", specification, "type") database_class = self.lookup_class(class_type) try: - database_object = database_class(specification) + database_object = database_class(specification, database_definition_object) except Exception as exception: raise RuntimeError( "unable to build database object for type %s" % (database_class) diff --git a/tests/my_database_definition.py b/tests/my_database_definition.py new file mode 100644 index 0000000..428edff --- /dev/null +++ b/tests/my_database_definition.py @@ -0,0 +1,35 @@ +import logging + +from tests.my_table_definition import MyTableDefinition + +logger = logging.getLogger(__name__) + + +# ---------------------------------------------------------------------------------------- +class MyDatabaseDefinition: + """ + Class which defines the database tables and revision migration path. + Used in concert with the normsql class. + """ + + # ---------------------------------------------------------------------------------------- + def __init__(self): + """ + Construct object. Do not connect to database. + """ + + self.LATEST_REVISION = 4 + + # ---------------------------------------------------------------------------------------- + async def apply_revision(self, database, revision): + + logger.debug(f"applying revision {revision}") + + # ---------------------------------------------------------------------------------------- + async def add_table_definitions(self, database): + """ + Make all the table definitions. + """ + + # Table schemas in our database. + database.add_table_definition(MyTableDefinition()) diff --git a/tests/test_database.py b/tests/test_database.py index d113bfa..33eb9dd 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -3,7 +3,9 @@ from dls_normsql.constants import ClassTypes, CommonFieldnames from dls_normsql.databases import Databases from tests.base_tester import BaseTester -from tests.my_table_definition import MyTableDefinition + +# from tests.my_table_definition import MyTableDefinition +from tests.my_database_definition import MyDatabaseDefinition logger = logging.getLogger(__name__) @@ -40,7 +42,7 @@ def test(self, logging_setup, output_directory): "type": ClassTypes.AIOMYSQL, "type_specific_tbd": { "database_name": "dls_normsql_pytest", - "host": "xchem-mysql", + "host": "docker-mysql", "port": 3306, "username": "root", "password": "root", @@ -63,11 +65,14 @@ class DatabaseTester(BaseTester): async def _main_coroutine(self, database_specification, output_directory): """ """ + database_definition_object = MyDatabaseDefinition() databases = Databases() - database1 = databases.build_object(database_specification) - database2 = databases.build_object(database_specification) - - database1.add_table_definition(MyTableDefinition()) + database1 = databases.build_object( + database_specification, database_definition_object + ) + database2 = databases.build_object( + database_specification, database_definition_object + ) try: # Connect to database. From 1bd257a8906da81126a65d90e2f8f7b4d9da2ce7 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 06:14:06 +0000 Subject: [PATCH 04/18] fix backup/restore test --- src/dls_normsql/aiomysql.py | 22 +++++++++++----------- src/dls_normsql/aiosqlite.py | 1 - tests/test_aiomysql.py | 2 +- tests/test_backup_restore.py | 9 ++++++--- tests/test_database.py | 2 -- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index c8ba410..0e245ff 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -8,7 +8,7 @@ import warnings from collections import OrderedDict from datetime import datetime -from typing import Any, List, Optional +from typing import Any, List import aiomysql @@ -26,6 +26,7 @@ connect_lock = asyncio.Lock() apply_revisions_lock = asyncio.Lock() + # ---------------------------------------------------------------------------------------- class Aiomysql: """ @@ -89,7 +90,7 @@ async def connect(self, should_drop_database=False): async with connect_lock: should_create_schemas = False - logger.debug(f"connecting to mysql") + logger.debug(f"connecting to mysql {self.__host}:{self.__port}") self.__connection = await aiomysql.connect( host=self.__host, @@ -362,20 +363,19 @@ async def insert( sql = self.__parameterize(sql, values_rows) + if why is None: + message = "\n%s\n%s" % (sql, values_rows) + else: + message = "%s:\n%s\n%s" % (why, sql, values_rows) + try: async with self.__connection.cursor() as cursor: await cursor.executemany(sql, values_rows) - if why is None: - logger.debug("\n%s\n%s" % (sql, values_rows)) - else: - logger.debug("%s:\n%s\n%s" % (why, sql, values_rows)) + logger.debug(message) - except (TypeError, aiomysql.OperationalError) as e: - if why is None: - raise RuntimeError(f"failed to execute {sql}") - else: - raise RuntimeError(f"failed to execute {why}: {sql}") + except (TypeError, aiomysql.OperationalError) as exception: + raise RuntimeError(f"{exception} doing {message}") # ---------------------------------------------------------------------------------------- async def update( diff --git a/src/dls_normsql/aiosqlite.py b/src/dls_normsql/aiosqlite.py index 1665596..2870fbf 100644 --- a/src/dls_normsql/aiosqlite.py +++ b/src/dls_normsql/aiosqlite.py @@ -8,7 +8,6 @@ import shutil from collections import OrderedDict from datetime import datetime -from typing import Optional import aiosqlite diff --git a/tests/test_aiomysql.py b/tests/test_aiomysql.py index a9d7ac3..9538e7b 100644 --- a/tests/test_aiomysql.py +++ b/tests/test_aiomysql.py @@ -33,7 +33,7 @@ class AiomysqlTester(BaseTester): async def _main_coroutine(self, database_specification, output_directory): """ """ pool = await aiomysql.create_pool( - host="xchem-mysql", + host="docker-mysql", port=3306, user="root", password="root", diff --git a/tests/test_backup_restore.py b/tests/test_backup_restore.py index 7285c58..9e85b9f 100644 --- a/tests/test_backup_restore.py +++ b/tests/test_backup_restore.py @@ -3,7 +3,7 @@ from dls_normsql.constants import ClassTypes, CommonFieldnames from dls_normsql.databases import Databases from tests.base_tester import BaseTester -from tests.my_table_definition import MyTableDefinition +from tests.my_database_definition import MyDatabaseDefinition logger = logging.getLogger(__name__) @@ -36,10 +36,13 @@ class DatabaseTester(BaseTester): async def _main_coroutine(self, database_specification, output_directory): """ """ + database_definition_object = MyDatabaseDefinition() databases = Databases() - database = databases.build_object(database_specification) - database.add_table_definition(MyTableDefinition()) + database = databases.build_object( + database_specification, + database_definition_object, + ) all_sql = "SELECT * FROM my_table" diff --git a/tests/test_database.py b/tests/test_database.py index 33eb9dd..ac6c1c8 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -3,8 +3,6 @@ from dls_normsql.constants import ClassTypes, CommonFieldnames from dls_normsql.databases import Databases from tests.base_tester import BaseTester - -# from tests.my_table_definition import MyTableDefinition from tests.my_database_definition import MyDatabaseDefinition logger = logging.getLogger(__name__) From e6fffe0c76df79f9a32ae50d9f344f5499c9fc58 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 06:51:47 +0000 Subject: [PATCH 05/18] fix wrong catch --- src/dls_normsql/aiomysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index 0e245ff..e244d16 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -112,7 +112,7 @@ async def connect(self, should_drop_database=False): try: await self.execute(f"USE {self.__database_name}") database_exists = True - except aiomysql.OperationalError: + except RuntimeError: database_exists = False if database_exists and should_drop_database: From 0d85d34a1a29cd1eab9135d158386171f7eaf6f2 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 08:00:51 +0000 Subject: [PATCH 06/18] mysql quit before close --- src/dls_normsql/aiomysql.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index e244d16..dcab615 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -109,6 +109,7 @@ async def connect(self, should_drop_database=False): f"SET GLOBAL TRANSACTION ISOLATION LEVEL {isolation_level}" ) + logger.debug(f"autocommit is {self.__connection.get_autocommit()}") try: await self.execute(f"USE {self.__database_name}") database_exists = True @@ -199,7 +200,11 @@ async def apply_revision(self, revision): async def disconnect(self): if self.__connection is not None: - logger.debug(f"[DISSHU] {callsign(self)} disconnecting") + # Commit final transaction if not currently autocommitting. + if not self.__connection.get_autocommit(): + logger.debug(f"[DISSHU] {callsign(self)} committing final transaction") + await self.__connection.commit() + logger.debug(f"[DISSHU] {callsign(self)} closing connection to server") self.__connection.close() self.__connection = None @@ -273,11 +278,11 @@ async def create_table(self, table): if isinstance(table, str): table = require("table definitions", self.__tables, table) - async with self.__connection.cursor() as cursor: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + await self.execute("DROP TABLE IF EXISTS %s" % (table.name)) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - await cursor.execute("DROP TABLE IF EXISTS %s" % (table.name)) + async with self.__connection.cursor() as cursor: fields_sql = [] indices_sql = [] @@ -371,7 +376,6 @@ async def insert( try: async with self.__connection.cursor() as cursor: await cursor.executemany(sql, values_rows) - logger.debug(message) except (TypeError, aiomysql.OperationalError) as exception: From f8fd8b6e897137608d669fb089f3e9379541eb81 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 18:38:13 +0100 Subject: [PATCH 07/18] back-ticks on table names --- src/dls_normsql/aiomysql.py | 6 +++--- src/dls_normsql/aiosqlite.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index dcab615..481eaf3 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -292,18 +292,18 @@ async def create_table(self, table): field_type = field["type"].upper() if field_type == "TEXT PRIMARY KEY": field_type = "VARCHAR(64) PRIMARY KEY" - fields_sql.append("%s %s" % (field_name, field_type)) + fields_sql.append("`%s` %s" % (field_name, field_type)) if field.get("index", False): if field_type == "TEXT": index_length = "(128)" else: index_length = "" indices_sql.append( - "CREATE INDEX %s_%s ON %s(%s%s)" + "CREATE INDEX `%s_%s` ON `%s`(`%s`%s)" % (table.name, field_name, table.name, field_name, index_length) ) - sql = "CREATE TABLE %s\n(%s)" % (table.name, ",\n ".join(fields_sql)) + sql = "CREATE TABLE `%s`\n(%s)" % (table.name, ",\n ".join(fields_sql)) logger.debug("\n%s\n%s" % (sql, "\n".join(indices_sql))) diff --git a/src/dls_normsql/aiosqlite.py b/src/dls_normsql/aiosqlite.py index 2870fbf..ab61a6f 100644 --- a/src/dls_normsql/aiosqlite.py +++ b/src/dls_normsql/aiosqlite.py @@ -282,14 +282,14 @@ async def create_table(self, table): for field_name in table.fields: field = table.fields[field_name] - fields_sql.append("%s %s" % (field_name, field["type"])) + fields_sql.append("`%s` %s" % (field_name, field["type"])) if field.get("index"): indices_sql.append( - "CREATE INDEX %s_%s ON %s(%s)" + "CREATE INDEX `%s_%s` ON `%s`(`%s`)" % (table.name, field_name, table.name, field_name) ) - sql = "CREATE TABLE %s\n(%s)" % (table.name, ",\n ".join(fields_sql)) + sql = "CREATE TABLE `%s`\n(%s)" % (table.name, ",\n ".join(fields_sql)) logger.debug("\n%s\n%s" % (sql, "\n".join(indices_sql))) From 00a651f93d8dd6863686b279daa535c8ca7ca87d Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 05:53:53 +0100 Subject: [PATCH 08/18] envvar for mysql access --- .github/workflows/code.yml | 11 ++++++++++- src/dls_normsql/aiomysql.py | 16 ++++++++++++++++ tests/test_aiomysql.py | 11 +++++++++-- tests/test_database.py | 11 +++++++++-- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 63c9248..95e325b 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -48,9 +48,19 @@ jobs: install: ".[dev,docs]" runs-on: ${{ matrix.os }} + + services: + mysql_service: + image: mysql:8.0 + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: root + env: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" + MYSQL_HOST: 127.0.0.1 steps: - name: Checkout @@ -211,5 +221,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_TOKEN }} - # dae_devops_fingerprint 39b6ade54ad4279a47cb3da654b3721e diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index 481eaf3..41bc6bf 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -14,6 +14,7 @@ # Utilities. from dls_utilpack.callsign import callsign +from dls_utilpack.envvar import Envvar from dls_utilpack.explain import explain from dls_utilpack.isodatetime import isodatetime_filename from dls_utilpack.require import require @@ -47,9 +48,24 @@ def __init__(self, specification, database_definition_object): # We might use values in type_specific_tbd during the connection method. self.__type_specific_tbd = t + # We will do environment variable substitution for host and port if they start with $. self.__host = require(s, t, "host") + if self.__host.startswith("$"): + envvar = Envvar(self.__host[1:]) + if not envvar.is_set: + raise RuntimeError( + f"configuration error: environment variable {self.__host[1:]} is not set" + ) + self.__host = envvar.value self.__port = require(s, t, "port") + if self.__port.startswith("$"): + envvar = Envvar(self.__port[1:], default=3306) + if not envvar.is_set: + raise RuntimeError( + f"configuration error: environment variable {self.__port[1:]} is not set" + ) + self.__port = int(envvar.value) self.__username = require(s, t, "username") diff --git a/tests/test_aiomysql.py b/tests/test_aiomysql.py index 9538e7b..564d8f0 100644 --- a/tests/test_aiomysql.py +++ b/tests/test_aiomysql.py @@ -1,6 +1,7 @@ import logging import aiomysql +from dls_utilpack.envvar import Envvar from tests.base_tester import BaseTester @@ -32,9 +33,15 @@ class AiomysqlTester(BaseTester): async def _main_coroutine(self, database_specification, output_directory): """ """ + + host = Envvar("MYSQL_HOST") + assert host.is_set + port = Envvar("MYSQL_PORT", default=3306) + assert port.is_set + pool = await aiomysql.create_pool( - host="docker-mysql", - port=3306, + host=host.value, + port=int(port.value), user="root", password="root", db="mysql", diff --git a/tests/test_database.py b/tests/test_database.py index ac6c1c8..61cce64 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,5 +1,7 @@ import logging +from dls_utilpack.envvar import Envvar + from dls_normsql.constants import ClassTypes, CommonFieldnames from dls_normsql.databases import Databases from tests.base_tester import BaseTester @@ -35,13 +37,18 @@ def test(self, logging_setup, output_directory): Tests the mysql implementation of Database. """ + host = Envvar("MYSQL_HOST") + assert host.is_set + port = Envvar("MYSQL_PORT", default=3306) + assert port.is_set + # Database specification. database_specification = { "type": ClassTypes.AIOMYSQL, "type_specific_tbd": { "database_name": "dls_normsql_pytest", - "host": "docker-mysql", - "port": 3306, + "host": "$MYSQL_HOST", + "port": "$MYSQL_PORT", "username": "root", "password": "root", }, From 786aa2de0f8cdd2fe54cc111c5ec9a2471a5e313 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:01:50 +0100 Subject: [PATCH 09/18] change MYSQL_HOST --- .github/workflows/code.yml | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 95e325b..0372790 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -60,7 +60,7 @@ jobs: env: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" - MYSQL_HOST: 127.0.0.1 + MYSQL_HOST: mysql_service steps: - name: Checkout diff --git a/pyproject.toml b/pyproject.toml index 4033cf2..4dbf0bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] description = "Normalized API over various sql libraries." -dependencies = ["dls-utilpack", "aiosqlite", "aiomysql"] +dependencies = ["dls-utilpack", "aiosqlite", "aiomysql", "cryptography"] dynamic = ["version"] license.file = "LICENSE" readme = "README.rst" @@ -102,4 +102,4 @@ source = ["src", "**/site-packages/"] [tool.tox] legacy_tox_ini = "[tox]\nskipsdist=True\n\n[testenv:{pre-commit,mypy,pytest,docs}]\n# Don't create a virtualenv for the command, requires tox-direct plugin\ndirect = True\npassenv = *\nallowlist_externals = \n pytest \n pre-commit\n mypy\n sphinx-build\n sphinx-autobuild\ncommands =\n pytest: pytest {posargs}\n mypy: mypy src tests {posargs}\n pre-commit: pre-commit run --all-files {posargs}\n docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html\n" -# dae_devops_fingerprint ba05247624b5791448c05976d419d99d +# dae_devops_fingerprint ecf3599f122f7e351e9e8366ada61f21 From 49e90af63f3aa8fda872750939bfa1906799b373 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:12:21 +0100 Subject: [PATCH 10/18] wait for mysql up --- .github/workflows/code.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 0372790..ebdc1d2 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -61,6 +61,7 @@ jobs: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" MYSQL_HOST: mysql_service + MYSQL_ROOT_PASSWORD: root steps: - name: Checkout @@ -79,6 +80,9 @@ jobs: - name: List dependency tree run: pipdeptree + - name: Wait for MySQL server to start + run: docker exec mysql_service mysqladmin --silent --wait=30 -uroot -p$MYSQL_ROOT_PASSWORD ping + - name: Run tests run: | sudo apt install environment-modules From 39da14b742f97bf693eee6efa8390e650421f1ab Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:17:57 +0100 Subject: [PATCH 11/18] health check options --- .github/workflows/code.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index ebdc1d2..a6e54f0 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -56,12 +56,12 @@ jobs: - 3306:3306 env: MYSQL_ROOT_PASSWORD: root + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 env: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" MYSQL_HOST: mysql_service - MYSQL_ROOT_PASSWORD: root steps: - name: Checkout @@ -80,9 +80,6 @@ jobs: - name: List dependency tree run: pipdeptree - - name: Wait for MySQL server to start - run: docker exec mysql_service mysqladmin --silent --wait=30 -uroot -p$MYSQL_ROOT_PASSWORD ping - - name: Run tests run: | sudo apt install environment-modules From 42cfaf86ee3bc9ac1e07c7a1440cafd558023c22 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:25:57 +0100 Subject: [PATCH 12/18] check docker ps --- .github/workflows/code.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index a6e54f0..6c4e3eb 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -56,7 +56,7 @@ jobs: - 3306:3306 env: MYSQL_ROOT_PASSWORD: root - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5 env: # https://github.com/pytest-dev/pytest/issues/2042 @@ -80,6 +80,9 @@ jobs: - name: List dependency tree run: pipdeptree + - name: Check MySQL service container status + run: docker ps + - name: Run tests run: | sudo apt install environment-modules From be51eaa23d55ec4a67243c8d7adb38c0074dfe57 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:33:18 +0100 Subject: [PATCH 13/18] reference mysql on localhost --- .github/workflows/code.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 6c4e3eb..d921443 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -53,7 +53,7 @@ jobs: mysql_service: image: mysql:8.0 ports: - - 3306:3306 + - 3306 env: MYSQL_ROOT_PASSWORD: root options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5 @@ -61,7 +61,8 @@ jobs: env: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" - MYSQL_HOST: mysql_service + MYSQL_HOST: 127.0.0.1 + MYSQL_PORT: ${{ job.services.mysql_service.ports[3306] }} steps: - name: Checkout From ccd3b8bf6dcae3e1d28819024f97b04a39b2b07e Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:36:25 +0100 Subject: [PATCH 14/18] misspelled jobs --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index d921443..0c13ae3 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -62,7 +62,7 @@ jobs: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: ${{ job.services.mysql_service.ports[3306] }} + MYSQL_PORT: ${{ jobs.services.mysql_service.ports[3306] }} steps: - name: Checkout From 23d2777cc498ce93fb630a853d7b9054507c20f6 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:42:22 +0100 Subject: [PATCH 15/18] put job.services inside step --- .github/workflows/code.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 0c13ae3..171c541 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -61,8 +61,6 @@ jobs: env: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" - MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: ${{ jobs.services.mysql_service.ports[3306] }} steps: - name: Checkout @@ -84,7 +82,13 @@ jobs: - name: Check MySQL service container status run: docker ps + - name: Start up the MySQL that comes with Unbuntu + run: sudo /etc/init.d/mysql start + - name: Run tests + env: + MYSQL_HOST: 127.0.0.1 + MYSQL_PORT: ${{ job.services.mysql_service.ports[3306] }} run: | sudo apt install environment-modules export MODULESHOME=/usr/share/modules From 6896f1482aea6b27666f2bb2f9eabe07ae0c95e1 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 06:48:42 +0100 Subject: [PATCH 16/18] use default port for in-built mysql --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 171c541..cc91592 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -88,7 +88,7 @@ jobs: - name: Run tests env: MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: ${{ job.services.mysql_service.ports[3306] }} + # MYSQL_PORT: ${{ job.services.mysql_service.ports[3306] }} run: | sudo apt install environment-modules export MODULESHOME=/usr/share/modules From 7a22466e84514fc3ca5e9307c3e8d052defbab93 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 07:51:55 +0100 Subject: [PATCH 17/18] new devops with mysql in it --- .dae-devops/Makefile | 4 ++-- .dae-devops/docs/conventions.rst | 4 ++-- .dae-devops/docs/developing.rst | 4 ++-- .dae-devops/docs/devops.rst | 4 ++-- .dae-devops/docs/docs_structure.rst | 4 ++-- .dae-devops/docs/installing.rst | 4 ++-- .dae-devops/docs/testing.rst | 4 ++-- .dae-devops/prepare_git_dependencies.sh | 4 ++-- .devcontainer/Dockerfile | 4 ++-- .devcontainer/devcontainer.json | 4 ++-- .github/CONTRIBUTING.rst | 4 ++-- .../actions/install_requirements/action.yml | 4 ++-- .github/dependabot.yml | 4 ++-- .github/pages/index.html | 4 ++-- .github/pages/make_switcher.py | 4 ++-- .github/workflows/code.yml | 24 ++++--------------- .github/workflows/docs.yml | 4 ++-- .github/workflows/docs_clean.yml | 4 ++-- .github/workflows/linkcheck.yml | 4 ++-- .gitlab-ci.yml | 4 ++-- docs/_static/css/custom.css | 4 ++-- docs/conf.py | 4 ++-- pyproject.toml | 4 ++-- src/dls_normsql/aiomysql.py | 2 +- 24 files changed, 50 insertions(+), 64 deletions(-) diff --git a/.dae-devops/Makefile b/.dae-devops/Makefile index 43aff4f..42f0739 100644 --- a/.dae-devops/Makefile +++ b/.dae-devops/Makefile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql # --------------------------------------------------------------------- @@ -69,4 +69,4 @@ publish_docs: cp -r build/html/* $(DOCS_PUBLISH_ROOT) -# dae_devops_fingerprint 7ad8e9ac2116461b84a0200de66ec86f +# dae_devops_fingerprint 62bfff1938ace30f8cb33ab30b9c60d5 diff --git a/.dae-devops/docs/conventions.rst b/.dae-devops/docs/conventions.rst index daa5ac9..3127ab6 100644 --- a/.dae-devops/docs/conventions.rst +++ b/.dae-devops/docs/conventions.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-normsql Naming conventions @@ -31,4 +31,4 @@ repository lowercase, hyphens -.. # dae_devops_fingerprint d57378dce1a17dd03d4629d169efd5ca +.. # dae_devops_fingerprint 8a72a4d0ee07d38513ef1273e9f9682e diff --git a/.dae-devops/docs/developing.rst b/.dae-devops/docs/developing.rst index 5e60b4b..e960c0f 100644 --- a/.dae-devops/docs/developing.rst +++ b/.dae-devops/docs/developing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-normsql Developing @@ -35,4 +35,4 @@ If you plan to modify the docs, you will need to:: -.. # dae_devops_fingerprint 06258b784a6611f9c5473dd7ab11d8c1 +.. # dae_devops_fingerprint 7b83e003f8bbbf54c27b4bd8141fb0e5 diff --git a/.dae-devops/docs/devops.rst b/.dae-devops/docs/devops.rst index fd67200..91ba7de 100644 --- a/.dae-devops/docs/devops.rst +++ b/.dae-devops/docs/devops.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-normsql Devops @@ -44,4 +44,4 @@ Publishing (for the Diamond intranet):: The Diamond intranet commands are not used for production. The production packaging and publishing are handled in the GitHub Actions workflows mechanism. -.. # dae_devops_fingerprint 315edec766c8c72d01b0fa9707eddd83 +.. # dae_devops_fingerprint d9ee7cc6b002b3508a4e55f96193c35e diff --git a/.dae-devops/docs/docs_structure.rst b/.dae-devops/docs/docs_structure.rst index 513c3e5..786e031 100644 --- a/.dae-devops/docs/docs_structure.rst +++ b/.dae-devops/docs/docs_structure.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-normsql About the documentation @@ -21,4 +21,4 @@ improve most documentation - often immensely. `More information on this topic. `_ -.. # dae_devops_fingerprint ec4b9c8af9a0c16a88c8afd2f10d8044 +.. # dae_devops_fingerprint eda941e8a0c3d217a601fb11b90f639b diff --git a/.dae-devops/docs/installing.rst b/.dae-devops/docs/installing.rst index 3a8530d..33fb463 100644 --- a/.dae-devops/docs/installing.rst +++ b/.dae-devops/docs/installing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-normsql Installing @@ -39,4 +39,4 @@ You can check the version that has been installed by typing:: $ dls-normsql --version $ dls-normsql --version-json -.. # dae_devops_fingerprint 9c071b85e47a709fdf94b44c012d3f03 +.. # dae_devops_fingerprint a2fe8dfbc79c151a5b899b244a529aa6 diff --git a/.dae-devops/docs/testing.rst b/.dae-devops/docs/testing.rst index 45f0b09..f093d8a 100644 --- a/.dae-devops/docs/testing.rst +++ b/.dae-devops/docs/testing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-normsql Testing @@ -31,4 +31,4 @@ This allows peeking in there to see what's been written by the test. -.. # dae_devops_fingerprint 16749d7794075d5edc250b7e3a37ae35 +.. # dae_devops_fingerprint 1801aaf6bd998d192c9fb4cdd6c3e79f diff --git a/.dae-devops/prepare_git_dependencies.sh b/.dae-devops/prepare_git_dependencies.sh index ae27539..c2ff2a9 100644 --- a/.dae-devops/prepare_git_dependencies.sh +++ b/.dae-devops/prepare_git_dependencies.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql me=${BASH_SOURCE} @@ -13,4 +13,4 @@ function __install { } -# dae_devops_fingerprint 1a34e24b1d5a45cb1ca98a9cbad72c6e \ No newline at end of file +# dae_devops_fingerprint b7ad9765b7da993eecef6a4ce23802c2 \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index bd945be..fd75a27 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql # This file is for use as a devcontainer and a runtime container @@ -40,4 +40,4 @@ ENV PATH=/venv/bin:$PATH ENTRYPOINT ["dls-normsql"] CMD ["--version"] -# dae_devops_fingerprint 6dc80a2a370a712769cdb9563fa17fc0 +# dae_devops_fingerprint a5c05253a7982ce4ee2e60abd1a254f0 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 16e1f3c..eb838a0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ // ********** Please don't edit this file! -// ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +// ********** It has been generated automatically by dae_devops version 0.5.3. // ********** For repository_name dls-normsql // For format details, see https://containers.dev/implementors/json_reference/ @@ -57,4 +57,4 @@ "postCreateCommand": "pip install -e .[dev]" } -// dae_devops_fingerprint 554efbeb9af2bdb211b7e057e098d375 +// dae_devops_fingerprint 1c59c18009b5b4578be9201b399a2b6c diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 77701e9..8566e2d 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-normsql Contributing to the project @@ -38,4 +38,4 @@ should follow. .. _Developer Guide: https://diamondlightsource.github.io/dls-normsql/main/developer/how-to/contribute.html -.. # dae_devops_fingerprint fbbfd64dec105165b3aa9295ebd7cd6e +.. # dae_devops_fingerprint 1e4da81da23bd1f1ea6d675517d6f9c5 diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index b71b4f1..7ae848a 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql name: Install requirements @@ -61,4 +61,4 @@ runs: shell: bash -# dae_devops_fingerprint 60c68e2129decf28ed4e20b91c02763e +# dae_devops_fingerprint 66e539b4856f80b66acfd95402e76d05 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 90d39b7..db5b6ad 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql # To get started with Dependabot version updates, you'll need to specify which @@ -19,4 +19,4 @@ updates: schedule: interval: "weekly" -# dae_devops_fingerprint c70363631c8e5cfa8a7053e5419dd1d4 +# dae_devops_fingerprint 61dd9f51dc2d4a0486ac8a76be91051f diff --git a/.github/pages/index.html b/.github/pages/index.html index 62121ad..3ed52a4 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -1,5 +1,5 @@ - + @@ -14,4 +14,4 @@ - + diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index 172c0d6..ded59e3 100644 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql import json @@ -102,4 +102,4 @@ def main(args=None): if __name__ == "__main__": main() -# dae_devops_fingerprint 82ea8a19d01c89a49e187624838f2126 +# dae_devops_fingerprint dd357c9cfe8e116f70c977b7d94d62db diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index cc91592..868c4fd 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql name: Code CI @@ -39,7 +39,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] # can add windows-latest, macos-latest - python: ["3.10", "3.11"] + python: ["3.10"] install: ["-e .[dev,docs]"] # Make one version be non-editable to test both paths of version code include: @@ -48,16 +48,6 @@ jobs: install: ".[dev,docs]" runs-on: ${{ matrix.os }} - - services: - mysql_service: - image: mysql:8.0 - ports: - - 3306 - env: - MYSQL_ROOT_PASSWORD: root - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5 - env: # https://github.com/pytest-dev/pytest/issues/2042 PY_IGNORE_IMPORTMISMATCH: "1" @@ -79,16 +69,11 @@ jobs: - name: List dependency tree run: pipdeptree - - name: Check MySQL service container status - run: docker ps - + # TODO: Make startup of MySQL able to be configured. - name: Start up the MySQL that comes with Unbuntu run: sudo /etc/init.d/mysql start - name: Run tests - env: - MYSQL_HOST: 127.0.0.1 - # MYSQL_PORT: ${{ job.services.mysql_service.ports[3306] }} run: | sudo apt install environment-modules export MODULESHOME=/usr/share/modules @@ -230,4 +215,5 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_TOKEN }} -# dae_devops_fingerprint 39b6ade54ad4279a47cb3da654b3721e + +# dae_devops_fingerprint d13e8289ca6252679ab7d9ae8db75617 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5109be2..ce8c669 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql name: Docs CI @@ -56,4 +56,4 @@ jobs: publish_dir: .github/pages keep_files: true -# dae_devops_fingerprint fc42ff4e21df624b69d71d1daf733ff7 +# dae_devops_fingerprint 0c3ce8bebad77dab099617eae89e2247 diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index 7f92cde..4d0672b 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql name: Docs Cleanup CI @@ -46,4 +46,4 @@ jobs: git commit -am "Removing redundant docs version $DOCS_VERSION" git push -# dae_devops_fingerprint 062f4bc18f25b11b80cbca161780c289 +# dae_devops_fingerprint b47d6e45e6ea9f929b8afe4f238d9473 diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 52c7a06..c394c5e 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql name: Link Check @@ -27,4 +27,4 @@ jobs: - name: Check links run: tox -e docs build -- -b linkcheck -# dae_devops_fingerprint 364f1f117764b2f547d91993c76ca046 +# dae_devops_fingerprint 674bb9fc122cc739ba92fe18045f0f1f diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5e9228..de67289 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql variables: @@ -87,4 +87,4 @@ package_pip: # # The validate_docs artifacts are in the build/html folder. # - make -f .dae-devops/Makefile publish_docs -# dae_devops_fingerprint b29bc99027c830817f5b2dd3c6c69c43 +# dae_devops_fingerprint f4eece0911ee4d3dbab40d858bc4962d diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index ac95ef8..63fc567 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,5 +1,5 @@ /* ********** Please don't edit this file! */ -/* ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. */ +/* ********** It has been generated automatically by dae_devops version 0.5.3. */ /* ********** For repository_name dls-normsql */ /* The theme normally has this, but I think it's ok to use the full width of the window in all @media sizes. @@ -15,4 +15,4 @@ max-width: 100%; } -/* dae_devops_fingerprint 6d64e77ebbc876fc118c61804d4caaba */ +/* dae_devops_fingerprint cadc619d1b64a206ac3a5e05c7b9d498 */ diff --git a/docs/conf.py b/docs/conf.py index 78cca0b..2a8bca7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql from pathlib import Path @@ -197,4 +197,4 @@ def setup(app): app.connect("source-read", ultimateReplace) -# dae_devops_fingerprint a5475bf5aabb8f8db72e43bae4457f5e +# dae_devops_fingerprint cd8b8b6c6077d960c9ed6af1579909b3 diff --git a/pyproject.toml b/pyproject.toml index 4dbf0bd..ded3f3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-normsql [build-system] @@ -102,4 +102,4 @@ source = ["src", "**/site-packages/"] [tool.tox] legacy_tox_ini = "[tox]\nskipsdist=True\n\n[testenv:{pre-commit,mypy,pytest,docs}]\n# Don't create a virtualenv for the command, requires tox-direct plugin\ndirect = True\npassenv = *\nallowlist_externals = \n pytest \n pre-commit\n mypy\n sphinx-build\n sphinx-autobuild\ncommands =\n pytest: pytest {posargs}\n mypy: mypy src tests {posargs}\n pre-commit: pre-commit run --all-files {posargs}\n docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html\n" -# dae_devops_fingerprint ecf3599f122f7e351e9e8366ada61f21 +# dae_devops_fingerprint ecc6232348dbe2bd0e46bde0c1961e3b diff --git a/src/dls_normsql/aiomysql.py b/src/dls_normsql/aiomysql.py index 41bc6bf..004f857 100644 --- a/src/dls_normsql/aiomysql.py +++ b/src/dls_normsql/aiomysql.py @@ -51,7 +51,7 @@ def __init__(self, specification, database_definition_object): # We will do environment variable substitution for host and port if they start with $. self.__host = require(s, t, "host") if self.__host.startswith("$"): - envvar = Envvar(self.__host[1:]) + envvar = Envvar(self.__host[1:], default="127.0.0.1") if not envvar.is_set: raise RuntimeError( f"configuration error: environment variable {self.__host[1:]} is not set" From 48b658be8ec9b737c21278439bfae17a3c68f317 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 07:58:34 +0100 Subject: [PATCH 18/18] default 127.0.0.1 for test host --- tests/test_aiomysql.py | 2 +- tests/test_database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_aiomysql.py b/tests/test_aiomysql.py index 564d8f0..5a97df4 100644 --- a/tests/test_aiomysql.py +++ b/tests/test_aiomysql.py @@ -34,7 +34,7 @@ class AiomysqlTester(BaseTester): async def _main_coroutine(self, database_specification, output_directory): """ """ - host = Envvar("MYSQL_HOST") + host = Envvar("MYSQL_HOST", default="127.0.0.1") assert host.is_set port = Envvar("MYSQL_PORT", default=3306) assert port.is_set diff --git a/tests/test_database.py b/tests/test_database.py index 61cce64..7343a51 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -37,7 +37,7 @@ def test(self, logging_setup, output_directory): Tests the mysql implementation of Database. """ - host = Envvar("MYSQL_HOST") + host = Envvar("MYSQL_HOST", default="127.0.0.1") assert host.is_set port = Envvar("MYSQL_PORT", default=3306) assert port.is_set