diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index b1f1d49e2..ef3daed9c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -24,7 +24,7 @@ body: Please copy and paste the code you were trying to run that caused the error. Feel free to include as little or as much as you think is relevant. This section will be automatically formatted into code, so no need for backticks. - render: shell + render: python validations: required: true - type: textarea @@ -41,8 +41,8 @@ body: attributes: label: Operating System options: - - Windows - macOS + - Windows - Linux validations: required: true @@ -60,10 +60,11 @@ body: attributes: label: Python Version options: - - "3.8" - - "3.9" - - "3.10" + - "3.12" - "3.11" + - "3.10" + - "3.9" + - "3.8" validations: required: true - type: textarea @@ -77,15 +78,11 @@ body: You can attach images or log files by clicking this area to highlight it and then dragging files in. If GitHub upload is not working, you can also copy and paste the output into this section. - - type: checkboxes - id: terms + - type: markdown attributes: - label: Code of Conduct - description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md) - options: - - label: I agree to follow this project's [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md) - required: true - - label: Have you checked the [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document? - required: true - - label: Have you ensured this bug was not already [reported](https://github.com/hdmf-dev/hdmf/issues)? - required: true + value: | + By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md). + + Before submitting this issue, please review the [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst). + + Please also ensure that this issue has not already been [reported](https://github.com/hdmf-dev/hdmf/issues). Thank you! diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml index 561a80cda..92a1e2cd2 100644 --- a/.github/ISSUE_TEMPLATE/documentation.yml +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -32,15 +32,11 @@ body: - No. validations: required: true - - type: checkboxes - id: terms + - type: markdown attributes: - label: Code of Conduct - description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md) - options: - - label: I agree to follow this project's [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md) - required: true - - label: Have you checked the [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document? - required: true - - label: Have you ensured this change was not already [requested](https://github.com/hdmf-dev/hdmf/issues)? - required: true + value: | + By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md). + + Before submitting this issue, please review the [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst). + + Please also ensure that this issue has not already been [reported](https://github.com/hdmf-dev/hdmf/issues). Thank you! diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 3fa8afbb1..f262928da 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -19,13 +19,10 @@ body: What are you trying to achieve with **HDMF**? Is this a more convenient way to do something that is already possible, or is a workaround currently unfeasible? + + If the change is related to a problem, please provide a clear and concise description of the problem. validations: required: true - - type: textarea - id: problem - attributes: - label: Is your feature request related to a problem? - description: A clear and concise description of what the problem is. - type: textarea id: solution attributes: @@ -50,15 +47,11 @@ body: - No. validations: required: true - - type: checkboxes - id: terms + - type: markdown attributes: - label: Code of Conduct - description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md) - options: - - label: I agree to follow this project's [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md) - required: true - - label: Have you checked the [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document? - required: true - - label: Have you ensured this change was not already [requested](https://github.com/hdmf-dev/hdmf/issues)? - required: true + value: | + By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md). + + Before submitting this issue, please review the [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst). + + Please also ensure that this issue has not already been [reported](https://github.com/hdmf-dev/hdmf/issues). Thank you! diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md index 60a725a73..11bd20bfa 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release.md +++ b/.github/PULL_REQUEST_TEMPLATE/release.md @@ -3,7 +3,9 @@ Prepare for release of HDMF [version] ### Before merging: - [ ] Major and minor releases: Update package versions in `requirements.txt`, `requirements-dev.txt`, `requirements-doc.txt`, `requirements-opt.txt`, and `environment-ros3.yml` to the latest versions, - and update dependency ranges in `pyproject.toml` and minimums in `requirements-min.txt` as needed + and update dependency ranges in `pyproject.toml` and minimums in `requirements-min.txt` as needed. + Run `pip install pur && pur -r requirements-dev.txt -r requirements.txt -r requirements-opt.txt` + and manually update `environment-ros3.yml`. - [ ] Check legal file dates and information in `Legal.txt`, `license.txt`, `README.rst`, `docs/source/conf.py`, and any other locations as needed - [ ] Update `pyproject.toml` as needed @@ -14,6 +16,8 @@ Prepare for release of HDMF [version] (`pytest && python test_gallery.py`) - [ ] Run PyNWB tests locally including gallery and validation tests, and inspect all warnings and outputs (`cd pynwb; python test.py -v > out.txt 2>&1`) +- [ ] Run HDMF-Zarr tests locally including gallery and validation tests, and inspect all warnings and outputs + (`cd hdmf-zarr; pytest && python test_gallery.py`) - [ ] Test docs locally and inspect all warnings and outputs `cd docs; make clean && make html` - [ ] Push changes to this PR and make sure all PRs to be included in this release have been merged - [ ] Check that the readthedocs build for this PR succeeds (build latest to pull the new branch, then activate and diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..24615639c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + # disable checking python requirements files because there are too + # many updates and dependabot will not ignore requirements-min.txt + # until https://github.com/dependabot/dependabot-core/issues/2883 is resolved + # workaround is to continue updating these files manually + + # - package-ecosystem: "pip" + # directory: "/" + # schedule: + # # Check for updates to requirements files and pyproject.toml every week + # interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2a1ebb784..edaa32a43 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,9 +9,7 @@ Show how to reproduce the new behavior (can be a bug fix or a new feature) ## Checklist -- [ ] Did you update CHANGELOG.md with your changes? -- [ ] Have you checked our [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document? -- [ ] Have you ensured the PR clearly describes the problem and the solution? -- [ ] Is your contribution compliant with our coding style? This can be checked running `ruff` from the source directory. -- [ ] Have you checked to ensure that there aren't other open [Pull Requests](https://github.com/hdmf-dev/hdmf/pulls) for the same change? -- [ ] Have you included the relevant issue number using "Fix #XXX" notation where XXX is the issue number? By including "Fix #XXX" you allow GitHub to close issue #XXX when the PR is merged. +- [ ] Did you update `CHANGELOG.md` with your changes? +- [ ] Does the PR clearly describe the problem and the solution? +- [ ] Have you reviewed our [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst)? +- [ ] Does the PR use "Fix #XXX" notation to tell GitHub to close the relevant issue numbered XXX when the PR is merged? diff --git a/.github/workflows/check_external_links.yml b/.github/workflows/check_external_links.yml index 031a26c1c..e030f37ae 100644 --- a/.github/workflows/check_external_links.yml +++ b/.github/workflows/check_external_links.yml @@ -8,22 +8,20 @@ on: jobs: check-external-links: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.11' # TODO update to 3.12 when optional reqs (e.g., oaklib) support 3.12 - name: Install Sphinx dependencies and package run: | diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 7aa79c9e7..32c4cc07f 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -1,19 +1,14 @@ ---- name: Codespell - on: - push: - branches: [dev] pull_request: - branches: [dev] + workflow_dispatch: jobs: codespell: name: Check for spelling errors runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 + - name: Checkout repo + uses: actions/checkout@v4 - name: Codespell uses: codespell-project/actions-codespell@v2 diff --git a/.github/workflows/deploy_release.yml b/.github/workflows/deploy_release.yml index ef9490f0e..66448cca2 100644 --- a/.github/workflows/deploy_release.yml +++ b/.github/workflows/deploy_release.yml @@ -10,15 +10,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo with submodules - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: 'recursive' fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install build dependencies run: | @@ -28,20 +28,20 @@ jobs: - name: Run tox tests run: | - tox -e py311-upgraded + tox -e py312-upgraded - name: Build wheel and source distribution run: | - tox -e build-py311-upgraded + tox -e build ls -1 dist - name: Test installation from a wheel run: | - tox -e wheelinstall --recreate --installpkg dist/*-none-any.whl + tox -e wheelinstall --installpkg dist/*-none-any.whl - name: Test installation from a source distribution run: | - tox -e wheelinstall --recreate --installpkg dist/*.tar.gz + tox -e wheelinstall --installpkg dist/*.tar.gz - name: Upload wheel and source distributions to PyPI run: | diff --git a/.github/workflows/project_action.yml b/.github/workflows/project_action.yml index 26195db02..bfca0b3f5 100644 --- a/.github/workflows/project_action.yml +++ b/.github/workflows/project_action.yml @@ -12,7 +12,7 @@ jobs: steps: - name: GitHub App token id: generate_token - uses: tibdex/github-app-token@v1.7.0 + uses: tibdex/github-app-token@v2 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PEM }} @@ -20,7 +20,7 @@ jobs: - name: Add to Developer Board env: TOKEN: ${{ steps.generate_token.outputs.token }} - uses: actions/add-to-project@v0.4.0 + uses: actions/add-to-project@v0.5.0 with: project-url: https://github.com/orgs/hdmf-dev/projects/7 github-token: ${{ env.TOKEN }} @@ -28,7 +28,7 @@ jobs: - name: Add to Community Board env: TOKEN: ${{ steps.generate_token.outputs.token }} - uses: actions/add-to-project@v0.4.0 + uses: actions/add-to-project@v0.5.0 with: project-url: https://github.com/orgs/hdmf-dev/projects/8 github-token: ${{ env.TOKEN }} diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 6d74fd2d9..1933fa75e 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -5,5 +5,7 @@ jobs: ruff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: chartboost/ruff-action@v1 + - name: Checkout repo + uses: actions/checkout@v4 + - name: Run ruff + uses: chartboost/ruff-action@v1 diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index 3e720f095..d5f8afc7f 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -18,45 +18,46 @@ jobs: defaults: run: shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: linux-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } - - { name: linux-python3.9 , test-tox-env: py39 , build-tox-env: build-py39 , python-ver: "3.9" , os: ubuntu-latest } - - { name: linux-python3.10 , test-tox-env: py310 , build-tox-env: build-py310 , python-ver: "3.10", os: ubuntu-latest } - - { name: linux-python3.11 , test-tox-env: py311 , build-tox-env: build-py311 , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-python3.11-optional , test-tox-env: py311-optional , build-tox-env: build-py311-optional , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-python3.11-prerelease , test-tox-env: py311-prerelease, build-tox-env: build-py311-prerelease, python-ver: "3.11", os: ubuntu-latest } - - { name: windows-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: windows-latest } - - { name: windows-python3.9 , test-tox-env: py39 , build-tox-env: build-py39 , python-ver: "3.9" , os: windows-latest } - - { name: windows-python3.10 , test-tox-env: py310 , build-tox-env: build-py310 , python-ver: "3.10", os: windows-latest } - - { name: windows-python3.11 , test-tox-env: py311 , build-tox-env: build-py311 , python-ver: "3.11", os: windows-latest } - - { name: windows-python3.11-optional , test-tox-env: py311-optional , build-tox-env: build-py311-optional , python-ver: "3.11", os: windows-latest } - - { name: windows-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: windows-latest } - - { name: windows-python3.11-prerelease, test-tox-env: py311-prerelease, build-tox-env: build-py311-prerelease, python-ver: "3.11", os: windows-latest } - - { name: macos-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: macos-latest } - - { name: macos-python3.9 , test-tox-env: py39 , build-tox-env: build-py39 , python-ver: "3.9" , os: macos-latest } - - { name: macos-python3.10 , test-tox-env: py310 , build-tox-env: build-py310 , python-ver: "3.10", os: macos-latest } - - { name: macos-python3.11 , test-tox-env: py311 , build-tox-env: build-py311 , python-ver: "3.11", os: macos-latest } - - { name: macos-python3.11-optional , test-tox-env: py311-optional , build-tox-env: build-py311-optional , python-ver: "3.11", os: macos-latest } - - { name: macos-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: macos-latest } - - { name: macos-python3.11-prerelease , test-tox-env: py311-prerelease, build-tox-env: build-py311-prerelease, python-ver: "3.11", os: macos-latest } + - { name: linux-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } + - { name: linux-python3.9 , test-tox-env: pytest-py39-pinned , python-ver: "3.9" , os: ubuntu-latest } + - { name: linux-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: ubuntu-latest } + - { name: linux-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: ubuntu-latest } + - { name: linux-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: ubuntu-latest } + - { name: linux-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: ubuntu-latest } + - { name: linux-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: ubuntu-latest } + - { name: linux-python3.12-prerelease , test-tox-env: pytest-py312-prerelease , python-ver: "3.12", os: ubuntu-latest } + - { name: windows-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: windows-latest } + - { name: windows-python3.9 , test-tox-env: pytest-py39-pinned , python-ver: "3.9" , os: windows-latest } + - { name: windows-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: windows-latest } + - { name: windows-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: windows-latest } + - { name: windows-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: windows-latest } + - { name: windows-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: windows-latest } + - { name: windows-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: windows-latest } + - { name: windows-python3.12-prerelease , test-tox-env: pytest-py312-prerelease , python-ver: "3.12", os: windows-latest } + - { name: macos-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: macos-latest } + - { name: macos-python3.9 , test-tox-env: pytest-py39-pinned , python-ver: "3.9" , os: macos-latest } + - { name: macos-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: macos-latest } + - { name: macos-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: macos-latest } + - { name: macos-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: macos-latest } + - { name: macos-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: macos-latest } + - { name: macos-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: macos-latest } + - { name: macos-python3.12-prerelease , test-tox-env: pytest-py312-prerelease , python-ver: "3.12", os: macos-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-ver }} @@ -72,16 +73,16 @@ jobs: - name: Build wheel and source distribution run: | - tox -e ${{ matrix.build-tox-env }} + tox -e build ls -1 dist - name: Test installation from a wheel run: | - tox -e wheelinstall --recreate --installpkg dist/*-none-any.whl + tox -e wheelinstall --installpkg dist/*-none-any.whl - name: Test installation from a source distribution run: | - tox -e wheelinstall --recreate --installpkg dist/*.tar.gz + tox -e wheelinstall --installpkg dist/*.tar.gz run-all-gallery-tests: name: ${{ matrix.name }} @@ -89,36 +90,34 @@ jobs: defaults: run: shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: linux-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } - - { name: linux-gallery-python3.11-optional , test-tox-env: gallery-py311-optional , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-gallery-python3.11-upgraded , test-tox-env: gallery-py311-upgraded , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-gallery-python3.11-prerelease , test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: ubuntu-latest } - - { name: windows-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: windows-latest } - - { name: windows-gallery-python3.11-optional , test-tox-env: gallery-py311-optional , python-ver: "3.11", os: windows-latest } - - { name: windows-gallery-python3.11-upgraded , test-tox-env: gallery-py311-upgraded , python-ver: "3.11", os: windows-latest } - - { name: windows-gallery-python3.11-prerelease, test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: windows-latest } - - { name: macos-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: macos-latest } - - { name: macos-gallery-python3.11-optional , test-tox-env: gallery-py311-optional , python-ver: "3.11", os: macos-latest } - - { name: macos-gallery-python3.11-upgraded , test-tox-env: gallery-py311-upgraded , python-ver: "3.11", os: macos-latest } - - { name: macos-gallery-python3.11-prerelease , test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: macos-latest } + - { name: linux-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } + - { name: linux-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: ubuntu-latest } + - { name: linux-gallery-python3.12-upgraded , test-tox-env: gallery-py312-upgraded , python-ver: "3.12", os: ubuntu-latest } + - { name: linux-gallery-python3.12-prerelease , test-tox-env: gallery-py312-prerelease , python-ver: "3.12", os: ubuntu-latest } + - { name: windows-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: windows-latest } + - { name: windows-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: windows-latest } + - { name: windows-gallery-python3.12-upgraded , test-tox-env: gallery-py312-upgraded , python-ver: "3.12", os: windows-latest } + - { name: windows-gallery-python3.12-prerelease, test-tox-env: gallery-py312-prerelease , python-ver: "3.12", os: windows-latest } + - { name: macos-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: macos-latest } + - { name: macos-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: macos-latest } + - { name: macos-gallery-python3.12-upgraded , test-tox-env: gallery-py312-upgraded , python-ver: "3.12", os: macos-latest } + - { name: macos-gallery-python3.12-prerelease , test-tox-env: gallery-py312-prerelease , python-ver: "3.12", os: macos-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-ver }} @@ -138,40 +137,41 @@ jobs: defaults: run: shell: bash -l {0} # needed for conda environment to work + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: conda-linux-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } - - { name: conda-linux-python3.9 , test-tox-env: py39 , build-tox-env: build-py39 , python-ver: "3.9" , os: ubuntu-latest } - - { name: conda-linux-python3.10 , test-tox-env: py310 , build-tox-env: build-py310 , python-ver: "3.10", os: ubuntu-latest } - - { name: conda-linux-python3.11 , test-tox-env: py311 , build-tox-env: build-py311 , python-ver: "3.11", os: ubuntu-latest } - - { name: conda-linux-python3.11-optional , test-tox-env: py311-optional , build-tox-env: build-py311-optional , python-ver: "3.11", os: ubuntu-latest } - - { name: conda-linux-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: ubuntu-latest } - - { name: conda-linux-python3.11-prerelease, test-tox-env: py311-prerelease, build-tox-env: build-py311-prerelease, python-ver: "3.11", os: ubuntu-latest } + - { name: conda-linux-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } + - { name: conda-linux-python3.9 , test-tox-env: pytest-py39-pinned , python-ver: "3.9" , os: ubuntu-latest } + - { name: conda-linux-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: ubuntu-latest } + - { name: conda-linux-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-linux-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-linux-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: ubuntu-latest } + - { name: conda-linux-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: ubuntu-latest } + - { name: conda-linux-python3.12-prerelease , test-tox-env: pytest-py312-prerelease , python-ver: "3.12", os: ubuntu-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-ver }} + channels: conda-forge + mamba-version: "*" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - conda install -c conda-forge tox + mamba install -c conda-forge "tox>=4" - name: Conda reporting run: | @@ -179,22 +179,23 @@ jobs: conda config --show-sources conda list --show-channel-urls + # NOTE tox installs packages from PyPI not conda-forge... - name: Run tox tests run: | tox -e ${{ matrix.test-tox-env }} - name: Build wheel and source distribution run: | - tox -e ${{ matrix.build-tox-env }} + tox -e build ls -1 dist - name: Test installation from a wheel run: | - tox -e wheelinstall --recreate --installpkg dist/*-none-any.whl + tox -e wheelinstall --installpkg dist/*-none-any.whl - name: Test installation from a source distribution run: | - tox -e wheelinstall --recreate --installpkg dist/*.tar.gz + tox -e wheelinstall --installpkg dist/*.tar.gz run-gallery-ros3-tests: name: ${{ matrix.name }} @@ -202,26 +203,25 @@ jobs: defaults: run: shell: bash -l {0} # necessary for conda + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: linux-gallery-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } - - { name: windows-gallery-python3.11-ros3, python-ver: "3.11", os: windows-latest } - - { name: macos-gallery-python3.11-ros3 , python-ver: "3.11", os: macos-latest } + - { name: linux-gallery-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } + - { name: windows-gallery-python3.12-ros3 , python-ver: "3.12", os: windows-latest } + - { name: macos-gallery-python3.12-ros3 , python-ver: "3.12", os: macos-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' + fetch-depth: 0 # tags are required to determine the version - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true activate-environment: ros3 @@ -229,6 +229,7 @@ jobs: python-version: ${{ matrix.python-ver }} channels: conda-forge auto-activate-base: false + mamba-version: "*" - name: Install run dependencies run: | diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index 051539aa6..4e43afc7b 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -19,6 +19,9 @@ jobs: defaults: run: shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.opt_req }} + cancel-in-progress: true strategy: matrix: include: @@ -28,21 +31,16 @@ jobs: - { os: macos-latest , opt_req: false } env: # used by codecov-action OS: ${{ matrix.os }} - PYTHON: '3.11' + PYTHON: '3.11' # TODO update to 3.12 when optional reqs (e.g., oaklib) support 3.12 steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON }} diff --git a/.github/workflows/run_hdmf_zarr_tests.yml b/.github/workflows/run_hdmf_zarr_tests.yml index 9221594f4..ecfdeaeeb 100644 --- a/.github/workflows/run_hdmf_zarr_tests.yml +++ b/.github/workflows/run_hdmf_zarr_tests.yml @@ -8,22 +8,20 @@ on: jobs: run-hdmf-zarr-tests: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.10' # use 3.10 until hdmf-zarr updates versioneer.py which breaks on newer python - name: Update pip run: python -m pip install --upgrade pip @@ -33,11 +31,11 @@ jobs: python -m pip list git clone https://github.com/hdmf-dev/hdmf-zarr.git --recurse-submodules cd hdmf-zarr - python -m pip install -r requirements-dev.txt -r requirements.txt + python -m pip install -r requirements-dev.txt # do not install the pinned install requirements # must install in editable mode for coverage to find sources - python -m pip install -e . # this will install a pinned version of hdmf instead of the current one + python -m pip install -e . # this will install a different version of hdmf from the current one cd .. - python -m pip uninstall -y hdmf # uninstall the pinned version of hdmf + python -m pip uninstall -y hdmf # uninstall the other version of hdmf python -m pip install . # reinstall current branch of hdmf python -m pip list diff --git a/.github/workflows/run_pynwb_tests.yml b/.github/workflows/run_pynwb_tests.yml index 2578e5383..bf3f32343 100644 --- a/.github/workflows/run_pynwb_tests.yml +++ b/.github/workflows/run_pynwb_tests.yml @@ -8,22 +8,20 @@ on: jobs: run-pynwb-tests: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Update pip run: python -m pip install --upgrade pip @@ -33,11 +31,11 @@ jobs: python -m pip list git clone https://github.com/NeurodataWithoutBorders/pynwb.git --recurse-submodules cd pynwb - python -m pip install -r requirements-dev.txt -r requirements.txt + python -m pip install -r requirements-dev.txt # do not install the pinned install requirements # must install in editable mode for coverage to find sources - python -m pip install -e . # this will install a pinned version of hdmf instead of the current one + python -m pip install -e . # this will install a different version of hdmf from the current one cd .. - python -m pip uninstall -y hdmf # uninstall the pinned version of hdmf + python -m pip uninstall -y hdmf # uninstall the other version of hdmf python -m pip install . # reinstall current branch of hdmf python -m pip list diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 8c7c437c3..049cec2e5 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -15,32 +15,30 @@ jobs: defaults: run: shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: linux-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } - - { name: linux-python3.11 , test-tox-env: py311 , build-tox-env: build-py311 , python-ver: "3.11", os: ubuntu-latest } # NOTE config below with "upload-wheels: true" specifies that wheels should be uploaded as an artifact - - { name: linux-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: ubuntu-latest , upload-wheels: true } - - { name: windows-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: windows-latest } - - { name: windows-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: windows-latest } - - { name: macos-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: macos-latest } - - { name: macos-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: macos-latest } + - { name: linux-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } + - { name: linux-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: ubuntu-latest } + - { name: linux-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: ubuntu-latest , upload-wheels: true } + - { name: windows-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: windows-latest } + - { name: windows-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: windows-latest } + - { name: macos-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: macos-latest } + - { name: macos-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: macos-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-ver }} @@ -56,20 +54,20 @@ jobs: - name: Build wheel and source distribution run: | - tox -e ${{ matrix.build-tox-env }} + tox -e build ls -1 dist - name: Test installation from a wheel run: | - tox -e wheelinstall --recreate --installpkg dist/*-none-any.whl + tox -e wheelinstall --installpkg dist/*-none-any.whl - name: Test installation from a source distribution run: | - tox -e wheelinstall --recreate --installpkg dist/*.tar.gz + tox -e wheelinstall --installpkg dist/*.tar.gz - name: Upload distribution as a workspace artifact if: ${{ matrix.upload-wheels }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: distributions path: dist @@ -80,27 +78,26 @@ jobs: defaults: run: shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: linux-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } - - { name: linux-gallery-python3.11-upgraded , test-tox-env: gallery-py311-upgraded, python-ver: "3.11", os: ubuntu-latest } - - { name: windows-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: windows-latest } - - { name: windows-gallery-python3.11-upgraded, test-tox-env: gallery-py311-upgraded, python-ver: "3.11", os: windows-latest } + - { name: linux-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } + - { name: linux-gallery-python3.12-upgraded , test-tox-env: gallery-py312-upgraded , python-ver: "3.12", os: ubuntu-latest } + - { name: windows-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: windows-latest } + - { name: windows-gallery-python3.12-upgraded , test-tox-env: gallery-py312-upgraded , python-ver: "3.12", os: windows-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-ver }} @@ -120,35 +117,35 @@ jobs: defaults: run: shell: bash -l {0} + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: conda-linux-python3.8-minimum , test-tox-env: py38-minimum , build-tox-env: build-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } - - { name: conda-linux-python3.11-upgraded , test-tox-env: py311-upgraded , build-tox-env: build-py311-upgraded , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-linux-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } + - { name: conda-linux-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: ubuntu-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' - fetch-depth: 0 # tags are required for versioneer to determine the version + fetch-depth: 0 # tags are required to determine the version - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-ver }} + channels: conda-forge + mamba-version: "*" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - conda install -c conda-forge tox + mamba install -c conda-forge "tox>=4" - name: Conda reporting run: | @@ -156,47 +153,46 @@ jobs: conda config --show-sources conda list --show-channel-urls + # NOTE tox installs packages from PyPI not conda-forge... - name: Run tox tests run: | tox -e ${{ matrix.test-tox-env }} - name: Build wheel and source distribution run: | - tox -e ${{ matrix.build-tox-env }} + tox -e build ls -1 dist - name: Test installation from a wheel run: | - tox -e wheelinstall --recreate --installpkg dist/*-none-any.whl + tox -e wheelinstall --installpkg dist/*-none-any.whl - name: Test installation from a source distribution run: | - tox -e wheelinstall --recreate --installpkg dist/*.tar.gz + tox -e wheelinstall --installpkg dist/*.tar.gz deploy-dev: name: Deploy pre-release from dev needs: [run-tests, run-gallery-tests, run-tests-on-conda] if: ${{ github.event_name == 'push' }} runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - name: Checkout repo with submodules - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: 'recursive' + fetch-depth: 0 # tags are required to determine the version - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Download wheel and source distributions from artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: distributions path: dist @@ -219,24 +215,23 @@ jobs: defaults: run: shell: bash -l {0} # necessary for conda + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true strategy: fail-fast: false matrix: include: - - { name: linux-gallery-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } + - { name: linux-gallery-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } steps: - - name: Cancel non-latest runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - all_but_latest: true - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: submodules: 'recursive' + fetch-depth: 0 # tags are required to determine the version - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true activate-environment: ros3 @@ -244,10 +239,10 @@ jobs: python-version: ${{ matrix.python-ver }} channels: conda-forge auto-activate-base: false + mamba-version: "*" - name: Install run dependencies run: | - pip install matplotlib pip install -e . pip list diff --git a/.mailmap b/.mailmap new file mode 100644 index 000000000..8babafefd --- /dev/null +++ b/.mailmap @@ -0,0 +1,4 @@ +# Mailmap is used by git to map author/committer names and/or E-Mail addresses +# See https://git-scm.com/docs/gitmailmap for details +Ben Dichter +Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb4908f7..a443ed0bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,41 @@ # HDMF Changelog -## HDMF 3.11.0 (Upcoming) +## HDMF 3.12.0 (Upcoming) + +### Enhancements +- Add Data.set_data_io(), which allows for setting a `DataIO` to a data object after-the-fact. @bendichter and @CodyCBakerPhD [#1013](https://github.com/hdmf-dev/hdmf/pull/1013) +- Added `add_ref_termset`, updated helper methods for `HERD`, revised `add_ref` to support validations prior to populating the tables + and added `add_ref_container`. @mavaylon1 [#968](https://github.com/hdmf-dev/hdmf/pull/968) +- Use `stacklevel` in most warnings. @rly [#1027](https://github.com/hdmf-dev/hdmf/pull/1027) + +### Minor Improvements +- Updated `__gather_columns` to ignore the order of bases when generating columns from the super class. @mavaylon1 [#991](https://github.com/hdmf-dev/hdmf/pull/991) +- Update `get_key` to return all the keys if there are multiple within a `HERD` instance. @mavaylon1 [#999](https://github.com/hdmf-dev/hdmf/pull/999) +- Improve HTML rendering of tables. @bendichter [#998](https://github.com/hdmf-dev/hdmf/pull/998) +- Improved issue and PR templates. @rly [#1004](https://github.com/hdmf-dev/hdmf/pull/1004) + +### Bug fixes +- Fixed issue with custom class generation when a spec has a `name`. @rly [#1006](https://github.com/hdmf-dev/hdmf/pull/1006) +- Fixed issue with usage of deprecated `ruamel.yaml.safe_load` in `src/hdmf/testing/validate_spec.py`. @rly [#1008](https://github.com/hdmf-dev/hdmf/pull/1008) +- Fixed issue where `ElementIdentifiers` data could be set to non-integer values. @rly [#1009](https://github.com/hdmf-dev/hdmf/pull/1009) +- Fixed issue where string datasets/attributes with isodatetime-formatted values failed validation against a text spec. @rly [#1026](https://github.com/hdmf-dev/hdmf/pull/1026) + +## HDMF 3.11.0 (October 30, 2023) ### Enhancements - Added `target_tables` attribute to `DynamicTable` to allow users to specify the target table of any predefined `DynamicTableRegion` columns of a `DynamicTable` subclass. @rly [#971](https://github.com/hdmf-dev/hdmf/pull/971) -- Updated `TermSet` to include `_repr_html_` for easy to read notebook representation. @mavaylon1 [967](https://github.com/hdmf-dev/hdmf/pull/967) +- Updated `TermSet` to include `_repr_html_` for easy to read notebook representation. @mavaylon1 [#967](https://github.com/hdmf-dev/hdmf/pull/967) + +### Minor improvements +- Set up GitHub dependabot to check for updates to GitHub Actions. @rly [#977](https://github.com/hdmf-dev/hdmf/pull/977) +- Simplify tox configuration. @rly [#988](https://github.com/hdmf-dev/hdmf/pull/988) +- Add testing for Python 3.12. @rly [#988](https://github.com/hdmf-dev/hdmf/pull/988) ### Bug fixes - Updated custom class generation to handle specs with fixed values and required names. @rly [#800](https://github.com/hdmf-dev/hdmf/pull/800) - Fixed custom class generation of `DynamicTable` subtypes to set attributes corresponding to column names for correct write. @rly [#800](https://github.com/hdmf-dev/hdmf/pull/800) +- Added a `.mailmap` file to correct mapping of names/emails in git logs. @oruebel [#976](https://github.com/hdmf-dev/hdmf/pull/976) ## HDMF 3.10.0 (October 3, 2023) diff --git a/README.rst b/README.rst index 7c4a24633..6717831b7 100644 --- a/README.rst +++ b/README.rst @@ -39,6 +39,9 @@ Overall Health .. image:: https://github.com/hdmf-dev/hdmf/actions/workflows/run_pynwb_tests.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf/actions/workflows/run_pynwb_tests.yml +.. image:: https://github.com/hdmf-dev/hdmf/actions/workflows/run_hdmf_zarr_tests.yml/badge.svg + :target: https://github.com/hdmf-dev/hdmf/actions/workflows/run_hdmf_zarr_tests.yml + .. image:: https://github.com/hdmf-dev/hdmf/actions/workflows/run_all_tests.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf/actions/workflows/run_all_tests.yml @@ -60,12 +63,12 @@ See the `HDMF documentation `_. By participating, you are expected to uphold this code. +This project and everyone participating in it is governed by our `code of conduct guidelines `_. By participating, you are expected to uphold this code. Contributing ============ -For details on how to contribute to HDMF see our `contribution guidelines `_. +For details on how to contribute to HDMF see our `contribution guidelines `_. Citing HDMF =========== diff --git a/docs/gallery/plot_external_resources.py b/docs/gallery/plot_external_resources.py index 3f7720d0b..5bf8dd5d8 100644 --- a/docs/gallery/plot_external_resources.py +++ b/docs/gallery/plot_external_resources.py @@ -91,6 +91,7 @@ # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnail_externalresources.png' from hdmf.common import HERD from hdmf.common import DynamicTable, VectorData +from hdmf.term_set import TermSet from hdmf import Container, HERDManager from hdmf import Data import numpy as np @@ -99,6 +100,13 @@ import warnings warnings.filterwarnings("ignore", category=UserWarning, message="HERD is experimental*") +try: + dir_path = os.path.dirname(os.path.abspath(__file__)) + yaml_file = os.path.join(dir_path, 'example_term_set.yaml') +except NameError: + dir_path = os.path.dirname(os.path.abspath('.')) + yaml_file = os.path.join(dir_path, 'gallery/example_term_set.yaml') + # Class to represent a file class HERDManagerContainer(Container, HERDManager): @@ -107,7 +115,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) -er = HERD() +herd = HERD() file = HERDManagerContainer(name='file') @@ -123,7 +131,8 @@ def __init__(self, **kwargs): # the underlying data structures accordingly. data = Data(name="species", data=['Homo sapiens', 'Mus musculus']) -er.add_ref( +data.parent = file +herd.add_ref( file=file, container=data, key='Homo sapiens', @@ -131,7 +140,7 @@ def __init__(self, **kwargs): entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=9606' ) -er.add_ref( +herd.add_ref( file=file, container=data, key='Mus musculus', @@ -156,7 +165,8 @@ def __init__(self, **kwargs): genotypes = DynamicTable(name='genotypes', description='My genotypes') genotypes.add_column(name='genotype_name', description="Name of genotypes") genotypes.add_row(id=0, genotype_name='Rorb') -er.add_ref( +genotypes.parent = file +herd.add_ref( file=file, container=genotypes, attribute='genotype_name', @@ -166,8 +176,8 @@ def __init__(self, **kwargs): ) # Note: :py:func:`~hdmf.common.resources.HERD.add_ref` internally resolves the object -# to the closest parent, so that ``er.add_ref(container=genotypes, attribute='genotype_name')`` and -# ``er.add_ref(container=genotypes.genotype_name, attribute=None)`` will ultimately both use the ``object_id`` +# to the closest parent, so that ``herd.add_ref(container=genotypes, attribute='genotype_name')`` and +# ``herd.add_ref(container=genotypes.genotype_name, attribute=None)`` will ultimately both use the ``object_id`` # of the ``genotypes.genotype_name`` :py:class:`~hdmf.common.table.VectorData` column and # not the object_id of the genotypes table. @@ -188,7 +198,7 @@ def __init__(self, **kwargs): species = DynamicTable(name='species', description='My species', columns=[col1]) species.parent = file -er.add_ref( +herd.add_ref( container=species, attribute='Species_Data', key='Ursus arctos horribilis', @@ -203,15 +213,15 @@ def __init__(self, **kwargs): # as separate tables. # `~hdmf.common.resources.HERD` as a flattened table -er.to_dataframe() +herd.to_dataframe() # The individual interlinked tables: -er.files.to_dataframe() -er.objects.to_dataframe() -er.entities.to_dataframe() -er.keys.to_dataframe() -er.object_keys.to_dataframe() -er.entity_keys.to_dataframe() +herd.files.to_dataframe() +herd.objects.to_dataframe() +herd.entities.to_dataframe() +herd.keys.to_dataframe() +herd.object_keys.to_dataframe() +herd.entity_keys.to_dataframe() ############################################################################### # Using the get_key method @@ -224,11 +234,11 @@ def __init__(self, **kwargs): # The :py:func:`~hdmf.common.resources.HERD.get_key` method will be able to return the # :py:class:`~hdmf.common.resources.Key` object if the :py:class:`~hdmf.common.resources.Key` object is unique. -genotype_key_object = er.get_key(key_name='Rorb') +genotype_key_object = herd.get_key(key_name='Rorb') # If the :py:class:`~hdmf.common.resources.Key` object has a duplicate name, then the user will need # to provide the unique (file, container, relative_path, field, key) combination. -species_key_object = er.get_key(file=file, +species_key_object = herd.get_key(file=file, container=species['Species_Data'], key_name='Ursus arctos horribilis') @@ -246,7 +256,7 @@ def __init__(self, **kwargs): # :py:func:`~hdmf.common.resources.HERD.add_ref` method. If a 'key_name' # is used, a new :py:class:`~hdmf.common.resources.Key` will be created. -er.add_ref( +herd.add_ref( file=file, container=genotypes, attribute='genotype_name', @@ -262,18 +272,18 @@ def __init__(self, **kwargs): # allows the user to retrieve all entities and key information associated with an `Object` in # the form of a pandas DataFrame. -er.get_object_entities(file=file, +herd.get_object_entities(file=file, container=genotypes['genotype_name'], relative_path='') ############################################################################### # Using the get_object_type # ------------------------------------------------------ -# The :py:class:`~hdmf.common.resources.HERD.get_object_entities` method +# The :py:func:`~hdmf.common.resources.HERD.get_object_entities` method # allows the user to retrieve all entities and key information associated with an `Object` in # the form of a pandas DataFrame. -er.get_object_type(object_type='Data') +herd.get_object_type(object_type='Data') ############################################################################### # Special Case: Using add_ref with compound data @@ -286,8 +296,7 @@ def __init__(self, **kwargs): # 'x' is using the external reference. # Let's create a new instance of :py:class:`~hdmf.common.resources.HERD`. -er = HERD() -file = HERDManagerContainer(name='file') +herd = HERD() data = Data( name='data_name', @@ -296,8 +305,9 @@ def __init__(self, **kwargs): dtype=[('species', 'U14'), ('age', 'i4'), ('weight', 'f4')] ) ) +data.parent = file -er.add_ref( +herd.add_ref( file=file, container=data, field='species', @@ -306,6 +316,45 @@ def __init__(self, **kwargs): entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=10090' ) +############################################################################### +# Using add_ref_termset +# ------------------------------------------------------ +# The :py:func:`~hdmf.common.resources.HERD.add_ref_termset` +# method allows users to not only validate terms, i.e., keys, but also +# add references for an entire datasets, rather than single entries as we saw +# prior with :py:func:`~hdmf.common.resources.HERD.add_ref`. + +# :py:func:`~hdmf.common.resources.HERD.add_ref_termset` has many optional fields, +# giving the user a range of control when adding references. Let's see an example. +herd = HERD() +terms = TermSet(term_schema_path=yaml_file) + +herd.add_ref_termset(file=file, + container=species, + attribute='Species_Data', + key='Ursus arctos horribilis', + termset=terms) + +############################################################################### +# Using add_ref_termset for an entire dataset +# ------------------------------------------------------ +# As mentioned above, :py:func:`~hdmf.common.resources.HERD.add_ref_termset` +# supports iteratively validating and populating :py:class:`~hdmf.common.resources.HERD`. + +# When populating :py:class:`~hdmf.common.resources.HERD`, users may have some terms +# that are not in the :py:class:`~hdmf.term_set.TermSet`. As a result, +# :py:func:`~hdmf.common.resources.HERD.add_ref_termset` will return all of the missing +# terms in a dictionary. It is up to the user to either add these terms to the +# :py:class:`~hdmf.term_set.TermSet` or remove them from the dataset. + +herd = HERD() +terms = TermSet(term_schema_path=yaml_file) + +herd.add_ref_termset(file=file, + container=species, + attribute='Species_Data', + termset=terms) + ############################################################################### # Write HERD # ------------------------------------------------------ @@ -313,7 +362,7 @@ def __init__(self, **kwargs): # the individual tables written to tsv. # The user provides the path, which contains the name of the file. -er.to_zip(path='./HERD.zip') +herd.to_zip(path='./HERD.zip') ############################################################################### # Read HERD diff --git a/docs/gallery/plot_term_set.py b/docs/gallery/plot_term_set.py index 86d53e553..71053bba5 100644 --- a/docs/gallery/plot_term_set.py +++ b/docs/gallery/plot_term_set.py @@ -190,3 +190,6 @@ # To add a column that is validated using :py:class:`~hdmf.term_set.TermSetWrapper`, # wrap the data in the :py:func:`~hdmf.common.table.DynamicTable.add_column` # method as if you were making a new instance of :py:class:`~hdmf.common.table.VectorData`. +species.add_column(name='Species_3', + description='...', + data=TermSetWrapper(value=['Ursus arctos horribilis', 'Mus musculus'], termset=terms),) diff --git a/docs/source/conf.py b/docs/source/conf.py index 0d43931e1..58fa3f2ba 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -70,7 +70,7 @@ } intersphinx_mapping = { - "python": ("https://docs.python.org/3.11", None), + "python": ("https://docs.python.org/3.12", None), "numpy": ("https://numpy.org/doc/stable/", None), "scipy": ("https://docs.scipy.org/doc/scipy/", None), "matplotlib": ("https://matplotlib.org/stable/", None), diff --git a/docs/source/install_developers.rst b/docs/source/install_developers.rst index 453ccf876..d043a351a 100644 --- a/docs/source/install_developers.rst +++ b/docs/source/install_developers.rst @@ -52,11 +52,11 @@ Option 2: Using conda The `conda package and environment management system`_ is an alternate way of managing virtual environments. First, install Anaconda_ to install the ``conda`` tool. Then create and -activate a new virtual environment called ``"hdmf-env"`` with Python 3.11 installed. +activate a new virtual environment called ``"hdmf-env"`` with Python 3.12 installed. .. code:: bash - conda create --name hdmf-env python=3.11 + conda create --name hdmf-env python=3.12 conda activate hdmf-env Similar to a virtual environment created with ``venv``, a conda environment diff --git a/docs/source/install_users.rst b/docs/source/install_users.rst index 6c0d235f2..8102651ff 100644 --- a/docs/source/install_users.rst +++ b/docs/source/install_users.rst @@ -4,7 +4,7 @@ Installing HDMF --------------- -HDMF requires having Python 3.8, 3.9, 3.10, or 3.11 installed. If you don't have Python installed and want the simplest way to +HDMF requires having Python 3.8, 3.9, 3.10, 3.11, or 3.12 installed. If you don't have Python installed and want the simplest way to get started, we recommend you install and use the `Anaconda Distribution`_. It includes Python, NumPy, and many other commonly used packages for scientific computing and data science. diff --git a/docs/source/software_process.rst b/docs/source/software_process.rst index 9ca706eb6..30501769e 100644 --- a/docs/source/software_process.rst +++ b/docs/source/software_process.rst @@ -19,7 +19,7 @@ inconsistencies. There are badges in the README_ file which shows the current condition of the dev branch. .. _GitHub Actions: https://github.com/hdmf-dev/hdmf/actions -.. _README: https://github.com/hdmf-dev/hdmf#readme +.. _README: https://github.com/hdmf-dev/hdmf/blob/dev/README.rst -------- diff --git a/environment-ros3.yml b/environment-ros3.yml index a8f2f0587..458b899ba 100644 --- a/environment-ros3.yml +++ b/environment-ros3.yml @@ -4,12 +4,12 @@ channels: - conda-forge - defaults dependencies: - - python==3.11 - - h5py==3.9.0 + - python==3.12 + - h5py==3.10.0 - matplotlib==3.8.0 - numpy==1.26.0 - - pandas==2.1.1 + - pandas==2.1.2 - python-dateutil==2.8.2 - - pytest==7.4.2 + - pytest==7.4.3 - pytest-cov==4.1.0 - setuptools diff --git a/requirements-dev.txt b/requirements-dev.txt index 760d48262..f61962728 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,12 +2,12 @@ # compute coverage, and create test environments. note that depending on the version of python installed, different # versions of requirements may be installed due to package incompatibilities. # -black==23.9.1 +black==23.10.1 codespell==2.2.6 coverage==7.3.2 -pre-commit==3.4.0 -pytest==7.4.2 +pre-commit==3.5.0 +pytest==7.4.3 pytest-cov==4.1.0 python-dateutil==2.8.2 -ruff==0.0.292 +ruff==0.1.3 tox==4.11.3 diff --git a/requirements.txt b/requirements.txt index df200c4ac..5182d5c2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # pinned dependencies to reproduce an entire development environment to use HDMF -h5py==3.9.0 -importlib-resources==6.0.0; python_version < "3.9" # TODO: remove when minimum python version is 3.9 +h5py==3.10.0 +importlib-resources==6.1.0; python_version < "3.9" # TODO: remove when minimum python version is 3.9 jsonschema==4.19.1 -numpy==1.26.0 -pandas==2.1.1 -ruamel.yaml==0.17.33 +numpy==1.26.1 +pandas==2.1.2 +ruamel.yaml==0.18.2 scipy==1.11.3 diff --git a/src/hdmf/backends/hdf5/h5_utils.py b/src/hdmf/backends/hdf5/h5_utils.py index 20de08033..85be494c2 100644 --- a/src/hdmf/backends/hdf5/h5_utils.py +++ b/src/hdmf/backends/hdf5/h5_utils.py @@ -499,7 +499,7 @@ def __init__(self, **kwargs): # Check for possible collision with other parameters if not isinstance(getargs('data', kwargs), Dataset) and self.__link_data: self.__link_data = False - warnings.warn('link_data parameter in H5DataIO will be ignored') + warnings.warn('link_data parameter in H5DataIO will be ignored', stacklevel=2) # Call the super constructor and consume the data parameter super().__init__(**kwargs) # Construct the dict with the io args, ignoring all options that were set to None @@ -523,7 +523,7 @@ def __init__(self, **kwargs): self.__iosettings.pop('compression', None) if 'compression_opts' in self.__iosettings: warnings.warn('Compression disabled by compression=False setting. ' + - 'compression_opts parameter will, therefore, be ignored.') + 'compression_opts parameter will, therefore, be ignored.', stacklevel=2) self.__iosettings.pop('compression_opts', None) # Validate the compression options used self._check_compression_options() @@ -537,7 +537,8 @@ def __init__(self, **kwargs): # Check possible parameter collisions if isinstance(self.data, Dataset): for k in self.__iosettings.keys(): - warnings.warn("%s in H5DataIO will be ignored with H5DataIO.data being an HDF5 dataset" % k) + warnings.warn("%s in H5DataIO will be ignored with H5DataIO.data being an HDF5 dataset" % k, + stacklevel=2) self.__dataset = None @@ -594,7 +595,7 @@ def _check_compression_options(self): if self.__iosettings['compression'] not in ['gzip', h5py_filters.h5z.FILTER_DEFLATE]: warnings.warn(str(self.__iosettings['compression']) + " compression may not be available " "on all installations of HDF5. Use of gzip is recommended to ensure portability of " - "the generated HDF5 files.") + "the generated HDF5 files.", stacklevel=3) @staticmethod def filter_available(filter, allow_plugin_filters): diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 5f445a3f5..643d9a7be 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -324,7 +324,9 @@ def copy_file(self, **kwargs): """ warnings.warn("The copy_file class method is no longer supported and may be removed in a future version of " - "HDMF. Please use the export method or h5py.File.copy method instead.", DeprecationWarning) + "HDMF. Please use the export method or h5py.File.copy method instead.", + category=DeprecationWarning, + stacklevel=2) source_filename, dest_filename, expand_external, expand_refs, expand_soft = getargs('source_filename', 'dest_filename', diff --git a/src/hdmf/backends/io.py b/src/hdmf/backends/io.py index 3d01c388b..4cd68e078 100644 --- a/src/hdmf/backends/io.py +++ b/src/hdmf/backends/io.py @@ -89,8 +89,8 @@ def write(self, **kwargs): from hdmf.common import HERD herd = HERD(type_map=self.manager.type_map) - # add_ref_term_set to search for and resolve the TermSetWrapper - herd.add_ref_term_set(container) # container would be the NWBFile + # add_ref_container to search for and resolve the TermSetWrapper + herd.add_ref_container(container) # container would be the NWBFile # write HERD herd.to_zip(path=self.herd_path) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 6a31f4cec..bdfbbc7da 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -225,8 +225,8 @@ def process_field_spec(cls, classdict, docval_args, parent_cls, attr_name, not_i fixed_value = getattr(field_spec, 'value', None) if fixed_value is not None: fields_conf['settable'] = False - if isinstance(field_spec, (BaseStorageSpec, LinkSpec)) and field_spec.data_type is not None: - # subgroups, datasets, and links with data types can have fixed names + if isinstance(field_spec, BaseStorageSpec) and field_spec.data_type is not None: + # subgroups and datasets with data types can have fixed names fixed_name = getattr(field_spec, 'name', None) if fixed_name is not None: fields_conf['required_name'] = fixed_name diff --git a/src/hdmf/build/map.py b/src/hdmf/build/map.py index 92b0c7499..5267609f5 100644 --- a/src/hdmf/build/map.py +++ b/src/hdmf/build/map.py @@ -4,4 +4,4 @@ import warnings warnings.warn('Classes in map.py should be imported from hdmf.build. Importing from hdmf.build.map will be removed ' - 'in HDMF 3.0.', DeprecationWarning) + 'in HDMF 3.0.', DeprecationWarning, stacklevel=2) diff --git a/src/hdmf/common/resources.py b/src/hdmf/common/resources.py index faead635f..f7f08b944 100644 --- a/src/hdmf/common/resources.py +++ b/src/hdmf/common/resources.py @@ -3,6 +3,8 @@ from . import register_class, EXP_NAMESPACE from . import get_type_map from ..container import Table, Row, Container, Data, AbstractContainer, HERDManager +from ..term_set import TermSet +from ..data_utils import DataIO from ..utils import docval, popargs, AllowPositional from ..build import TypeMap from ..term_set import TermSetWrapper @@ -10,6 +12,7 @@ import os import zipfile from collections import namedtuple +from warnings import warn class KeyTable(Table): @@ -358,16 +361,17 @@ def _check_object_field(self, **kwargs): relative_path = kwargs['relative_path'] field = kwargs['field'] create = kwargs['create'] + file_object_id = file.object_id files_idx = self.files.which(file_object_id=file_object_id) - if len(files_idx) > 1: + if len(files_idx) > 1: # pragma: no cover + # It isn't possible for len(files_idx) > 1 without the user directly using _add_file raise ValueError("Found multiple instances of the same file.") elif len(files_idx) == 1: files_idx = files_idx[0] else: - self._add_file(file_object_id) - files_idx = self.files.which(file_object_id=file_object_id)[0] + files_idx = None objecttable_idx = self.objects.which(object_id=container.object_id) @@ -378,10 +382,16 @@ def _check_object_field(self, **kwargs): if len(objecttable_idx) == 1: return self.objects.row[objecttable_idx[0]] elif len(objecttable_idx) == 0 and create: - return self._add_object(files_idx=files_idx, container=container, relative_path=relative_path, field=field) + # Used for add_ref + return {'file_object_id': file_object_id, + 'files_idx': files_idx, + 'container': container, + 'relative_path': relative_path, + 'field': field} elif len(objecttable_idx) == 0 and not create: raise ValueError("Object not in Object Table.") - else: + else: # pragma: no cover + # It isn't possible for this to happen unless the user used _add_object. raise ValueError("Found multiple instances of the same object id, relative path, " "and field in objects table.") @@ -437,7 +447,7 @@ def __check_termset_wrapper(self, **kwargs): @docval({'name': 'root_container', 'type': HERDManager, 'doc': 'The root container or file containing objects with a TermSet.'}) - def add_ref_term_set(self, **kwargs): + def add_ref_container(self, **kwargs): """ Method to search through the root_container for all instances of TermSet. Currently, only datasets are supported. By using a TermSet, the data comes validated @@ -466,63 +476,73 @@ def add_ref_term_set(self, **kwargs): entity_id=entity_id, entity_uri=entity_uri) - @docval({'name': 'key_name', 'type': str, 'doc': 'The name of the Key to get.'}, - {'name': 'file', 'type': HERDManager, 'doc': 'The file associated with the container.', + @docval({'name': 'file', 'type': HERDManager, 'doc': 'The file associated with the container.', 'default': None}, {'name': 'container', 'type': (str, AbstractContainer), 'default': None, 'doc': ('The Container/Data object that uses the key or ' - 'the object id for the Container/Data object that uses the key.')}, - {'name': 'relative_path', 'type': str, - 'doc': ('The relative_path of the attribute of the object that uses ', - 'an external resource reference key. Use an empty string if not applicable.'), - 'default': ''}, + 'the object_id for the Container/Data object that uses the key.')}, + {'name': 'attribute', 'type': str, + 'doc': 'The attribute of the container for the external reference.', 'default': None}, {'name': 'field', 'type': str, 'default': '', - 'doc': ('The field of the compound data type using an external resource.')}) - def get_key(self, **kwargs): + 'doc': ('The field of the compound data type using an external resource.')}, + {'name': 'key', 'type': (str, Key), 'default': None, + 'doc': 'The name of the key or the Key object from the KeyTable for the key to add a resource for.'}, + {'name': 'termset', 'type': TermSet, + 'doc': 'The TermSet to be used if the container/attribute does not have one.'} + ) + def add_ref_termset(self, **kwargs): """ - Return a Key. - - If container, relative_path, and field are provided, the Key that corresponds to the given name of the key - for the given container, relative_path, and field is returned. + This method allows users to take advantage of using the TermSet class to provide the entity information + for add_ref, while also validating the data. This method supports adding a single key or an entire dataset + to the HERD tables. For both cases, the term, i.e., key, will be validated against the permissible values + in the TermSet. If valid, it will proceed to call add_ref. Otherwise, the method will return a dict of + missing terms (terms not found in the TermSet). """ - key_name, container, relative_path, field = popargs('key_name', 'container', 'relative_path', 'field', kwargs) - key_idx_matches = self.keys.which(key=key_name) - file = kwargs['file'] + container = kwargs['container'] + attribute = kwargs['attribute'] + key = kwargs['key'] + field = kwargs['field'] + termset = kwargs['termset'] - if container is not None: - if file is None: - file = self._get_file_from_container(container=container) - # if same key is used multiple times, determine - # which instance based on the Container - object_field = self._check_object_field(file=file, - container=container, - relative_path=relative_path, - field=field) - for row_idx in self.object_keys.which(objects_idx=object_field.idx): - key_idx = self.object_keys['keys_idx', row_idx] - if key_idx in key_idx_matches: - return self.keys.row[key_idx] - msg = "No key found with that container." - raise ValueError(msg) + if file is None: + file = self._get_file_from_container(container=container) + # if key is provided then add_ref proceeds as normal + if key is not None: + data = [key] else: - if len(key_idx_matches) == 0: - # the key has never been used before - raise ValueError("key '%s' does not exist" % key_name) - elif len(key_idx_matches) > 1: - msg = "There are more than one key with that name. Please search with additional information." - raise ValueError(msg) + # if the key is not provided, proceed to "bulk add" + if attribute is None: + data_object = container else: - return self.keys.row[key_idx_matches[0]] - - @docval({'name': 'entity_id', 'type': str, 'doc': 'The ID for the identifier at the resource.'}) - def get_entity(self, **kwargs): - entity_id = kwargs['entity_id'] - entity = self.entities.which(entity_id=entity_id) - if len(entity)>0: - return self.entities.row[entity[0]] - else: - return None + data_object = getattr(container, attribute) + if isinstance(data_object, (Data, DataIO)): + data = data_object.data + elif isinstance(data_object, (list, tuple, np.ndarray)): + data = data_object + else: + msg = ("The data object being used is not supported. " + "Please review the documentation for supported types.") + raise ValueError(msg) + missing_terms = [] + for term in data: + # check the data according to the permissible_values + try: + term_info = termset[term] + except ValueError: + missing_terms.append(term) + continue + entity_id = term_info[0] + entity_uri = term_info[2] + self.add_ref(file=file, + container=container, + attribute=attribute, + key=term, + field=field, + entity_id=entity_id, + entity_uri=entity_uri) + if len(missing_terms)>0: + return {"missing_terms": missing_terms} @docval({'name': 'container', 'type': (str, AbstractContainer), 'default': None, 'doc': ('The Container/Data object that uses the key or ' @@ -550,6 +570,7 @@ def add_ref(self, **kwargs): container = kwargs['container'] attribute = kwargs['attribute'] if isinstance(container, Data): + # Used when using the TermSetWrapper if attribute == 'data': attribute = None key = kwargs['key'] @@ -558,9 +579,60 @@ def add_ref(self, **kwargs): entity_uri = kwargs['entity_uri'] file = kwargs['file'] + ################## + # Set File if None + ################## if file is None: file = self._get_file_from_container(container=container) + # TODO: Add this once you've created a HDMF_file to rework testing + # else: + # file_from_container = self._get_file_from_container(container=container) + # if file.object_id != file_from_container.object_id: + # msg = "The file given does not match the file in which the container is stored." + # raise ValueError(msg) + + ################ + # Set Key Checks + ################ + add_key = False + add_object_key = False + check_object_key = False + if not isinstance(key, Key): + add_key = True + add_object_key = True + else: + # Check to see that the existing key is being used with the object. + # If true, do nothing. If false, create a new obj/key relationship + # in the ObjectKeyTable + check_object_key = True + + ################### + # Set Entity Checks + ################### + add_entity_key = False + add_entity = False + + entity = self.get_entity(entity_id=entity_id) + check_entity_key = False + if entity is None: + if entity_uri is None: + msg = 'New entities must have an entity_uri.' + raise ValueError(msg) + + add_entity = True + add_entity_key = True + else: + # The entity exists and so we need to check if an entity_key exists + # for this entity and key combination. + check_entity_key = True + if entity_uri is not None: + entity_uri = entity.entity_uri + msg = 'This entity already exists. Ignoring new entity uri' + warn(msg, stacklevel=2) + ################# + # Validate Object + ################# if attribute is None: # Trivial Case relative_path = '' object_field = self._check_object_field(file=file, @@ -605,69 +677,142 @@ def add_ref(self, **kwargs): relative_path=relative_path, field=field) - if not isinstance(key, Key): + ####################################### + # Validate Parameters and Populate HERD + ####################################### + if isinstance(object_field, dict): + # Create the object and file + if object_field['files_idx'] is None: + self._add_file(object_field['file_object_id']) + object_field['files_idx'] = self.files.which(file_object_id=object_field['file_object_id'])[0] + object_field = self._add_object(files_idx=object_field['files_idx'], + container=object_field['container'], + relative_path=object_field['relative_path'], + field=object_field['field']) + + if add_key: + # Now that object_field is set, we need to check if + # the key has been associated with that object. + # If so, just reuse the key. + key_exists = False key_idx_matches = self.keys.which(key=key) - # if same key is used multiple times, determine - # which instance based on the Container - for row_idx in self.object_keys.which(objects_idx=object_field.idx): - key_idx = self.object_keys['keys_idx', row_idx] - if key_idx in key_idx_matches: - msg = "Use Key Object when referencing an existing (container, relative_path, key)" - raise ValueError(msg) - - key = self._add_key(key) - self._add_object_key(object_field, key) - - else: - # Check to see that the existing key is being used with the object. - # If true, do nothing. If false, create a new obj/key relationship - # in the ObjectKeyTable + if len(key_idx_matches)!=0: + for row_idx in self.object_keys.which(objects_idx=object_field.idx): + key_idx = self.object_keys['keys_idx', row_idx] + if key_idx in key_idx_matches: + key_exists = True # Make sure we don't add the key. + # Automatically resolve the key for keys associated with + # the same object. + key = self.keys.row[key_idx] + + if not key_exists: + key = self._add_key(key) + + if check_object_key: + # When using a Key Object, we want to still check for whether the key + # has been used with the Object object. If not, add it to ObjectKeyTable. + # If so, do nothing and add_object_key remains False. + obj_key_exists = False key_idx = key.idx object_key_row_idx = self.object_keys.which(keys_idx=key_idx) if len(object_key_row_idx)!=0: - obj_key_check = False + # this means there exists rows where the key is in the ObjectKeyTable for row_idx in object_key_row_idx: obj_idx = self.object_keys['objects_idx', row_idx] if obj_idx == object_field.idx: - obj_key_check = True - if not obj_key_check: - self._add_object_key(object_field, key) - else: - msg = "Cannot find key object. Create new Key with string." - raise ValueError(msg) - # check if the key and object have been related in the ObjectKeyTable + obj_key_exists = True + # this means there is already a object-key relationship recorded + if not obj_key_exists: + # this means that though the key is there, there is no object-key relationship + add_object_key = True - entity = self.get_entity(entity_id=entity_id) - if entity is None: - if entity_uri is None: - msg = 'New entities must have an entity_uri.' - raise ValueError(msg) - entity = self._add_entity(entity_id, entity_uri) - self._add_entity_key(entity, key) - else: - if entity_uri is not None: - msg = 'If you plan on reusing an entity, then entity_uri parameter must be None.' - raise ValueError(msg) + if add_object_key: + self._add_object_key(object_field, key) + + if check_entity_key: # check for entity-key relationship in EntityKeyTable + entity_key_check = False key_idx = key.idx entity_key_row_idx = self.entity_keys.which(keys_idx=key_idx) if len(entity_key_row_idx)!=0: # this means there exists rows where the key is in the EntityKeyTable - entity_key_check = False for row_idx in entity_key_row_idx: entity_idx = self.entity_keys['entities_idx', row_idx] if entity_idx == entity.idx: entity_key_check = True - # this means there is already a key-entity relationship recorded + # this means there is already a entity-key relationship recorded if not entity_key_check: - # this means that though the key is there, there is not key-entity relationship - # a.k.a add it now - self._add_entity_key(entity, key) + # this means that though the key is there, there is no entity-key relationship + add_entity_key = True else: # this means that specific key is not in the EntityKeyTable, so add it and establish # the relationship with the entity - self._add_entity_key(entity, key) - return key, entity + add_entity_key = True + + if add_entity: + entity = self._add_entity(entity_id, entity_uri) + + if add_entity_key: + self._add_entity_key(entity, key) + + @docval({'name': 'key_name', 'type': str, 'doc': 'The name of the Key to get.'}, + {'name': 'file', 'type': HERDManager, 'doc': 'The file associated with the container.', + 'default': None}, + {'name': 'container', 'type': (str, AbstractContainer), 'default': None, + 'doc': ('The Container/Data object that uses the key or ' + 'the object id for the Container/Data object that uses the key.')}, + {'name': 'relative_path', 'type': str, + 'doc': ('The relative_path of the attribute of the object that uses ', + 'an external resource reference key. Use an empty string if not applicable.'), + 'default': ''}, + {'name': 'field', 'type': str, 'default': '', + 'doc': ('The field of the compound data type using an external resource.')}) + def get_key(self, **kwargs): + """ + Return a Key. + + If container, relative_path, and field are provided, the Key that corresponds to the given name of the key + for the given container, relative_path, and field is returned. + + If there are multiple matches, a list of all matching keys will be returned. + """ + key_name, container, relative_path, field = popargs('key_name', 'container', 'relative_path', 'field', kwargs) + key_idx_matches = self.keys.which(key=key_name) + + file = kwargs['file'] + + if container is not None: + if file is None: + file = self._get_file_from_container(container=container) + # if same key is used multiple times, determine + # which instance based on the Container + object_field = self._check_object_field(file=file, + container=container, + relative_path=relative_path, + field=field) + for row_idx in self.object_keys.which(objects_idx=object_field.idx): + key_idx = self.object_keys['keys_idx', row_idx] + if key_idx in key_idx_matches: + return self.keys.row[key_idx] + msg = "No key found with that container." + raise ValueError(msg) + else: + if len(key_idx_matches) == 0: + # the key has never been used before + raise ValueError("key '%s' does not exist" % key_name) + elif len(key_idx_matches) > 1: + return [self.keys.row[x] for x in key_idx_matches] + else: + return self.keys.row[key_idx_matches[0]] + + @docval({'name': 'entity_id', 'type': str, 'doc': 'The ID for the identifier at the resource.'}) + def get_entity(self, **kwargs): + entity_id = kwargs['entity_id'] + entity = self.entities.which(entity_id=entity_id) + if len(entity)>0: + return self.entities.row[entity[0]] + else: + return None @docval({'name': 'object_type', 'type': str, 'doc': 'The type of the object. This is also the parent in relative_path.'}, diff --git a/src/hdmf/common/table.py b/src/hdmf/common/table.py index 58f0470e1..5eeedcd86 100644 --- a/src/hdmf/common/table.py +++ b/src/hdmf/common/table.py @@ -15,7 +15,7 @@ from . import register_class, EXP_NAMESPACE from ..container import Container, Data from ..data_utils import DataIO, AbstractDataChunkIterator -from ..utils import docval, getargs, ExtenderMeta, popargs, pystr, AllowPositional +from ..utils import docval, getargs, ExtenderMeta, popargs, pystr, AllowPositional, check_type from ..term_set import TermSetWrapper @@ -211,8 +211,8 @@ class ElementIdentifiers(Data): """ @docval({'name': 'name', 'type': str, 'doc': 'the name of this ElementIdentifiers'}, - {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'a 1D dataset containing identifiers', - 'default': list()}, + {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'a 1D dataset containing integer identifiers', + 'default': list(), 'shape': (None,)}, allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): super().__init__(**kwargs) @@ -237,6 +237,20 @@ def __eq__(self, other): # Find all matching locations return np.in1d(self.data, search_ids).nonzero()[0] + def _validate_new_data(self, data): + # NOTE this may not cover all the many AbstractDataChunkIterator edge cases + if (isinstance(data, AbstractDataChunkIterator) or + (hasattr(data, "data") and isinstance(data.data, AbstractDataChunkIterator))): + if not np.issubdtype(data.dtype, np.integer): + raise ValueError("ElementIdentifiers must contain integers") + elif hasattr(data, "__len__") and len(data): + self._validate_new_data_element(data[0]) + + def _validate_new_data_element(self, arg): + if not check_type(arg, int): + raise ValueError("ElementIdentifiers must contain integers") + super()._validate_new_data_element(arg) + @register_class('DynamicTable') class DynamicTable(Container): @@ -278,11 +292,17 @@ def __gather_columns(cls, name, bases, classdict): msg = "'__columns__' must be of type tuple, found %s" % type(cls.__columns__) raise TypeError(msg) - if (len(bases) and 'DynamicTable' in globals() and issubclass(bases[-1], Container) - and bases[-1].__columns__ is not cls.__columns__): - new_columns = list(cls.__columns__) - new_columns[0:0] = bases[-1].__columns__ # prepend superclass columns to new_columns - cls.__columns__ = tuple(new_columns) + if len(bases) and 'DynamicTable' in globals(): + for item in bases[::-1]: # look for __columns__ in the base classes, closest first + if issubclass(item, Container): + try: + if item.__columns__ is not cls.__columns__: + new_columns = list(cls.__columns__) + new_columns[0:0] = item.__columns__ # prepend superclass columns to new_columns + cls.__columns__ = tuple(new_columns) + break + except AttributeError: # raises error when "__columns__" is not an attr of item + continue @docval({'name': 'name', 'type': str, 'doc': 'the name of this table'}, # noqa: C901 {'name': 'description', 'type': str, 'doc': 'a description of what is in this table'}, @@ -483,7 +503,7 @@ def __set_table_attr(self, col): msg = ("An attribute '%s' already exists on %s '%s' so this column cannot be accessed as an attribute, " "e.g., table.%s; it can only be accessed using other methods, e.g., table['%s']." % (col.name, self.__class__.__name__, self.name, col.name, col.name)) - warn(msg) + warn(msg, stacklevel=2) else: setattr(self, col.name, col) @@ -744,7 +764,7 @@ def add_column(self, **kwargs): # noqa: C901 if isinstance(index, VectorIndex): warn("Passing a VectorIndex in for index may lead to unexpected behavior. This functionality will be " - "deprecated in a future version of HDMF.", FutureWarning) + "deprecated in a future version of HDMF.", category=FutureWarning, stacklevel=2) if name in self.__colids: # column has already been added msg = "column '%s' already exists in %s '%s'" % (name, self.__class__.__name__, self.name) @@ -761,7 +781,7 @@ def add_column(self, **kwargs): # noqa: C901 "Please ensure the new column complies with the spec. " "This will raise an error in a future version of HDMF." % (name, self.__class__.__name__, spec_table)) - warn(msg) + warn(msg, stacklevel=2) index_bool = index or not isinstance(index, bool) spec_index = self.__uninit_cols[name].get('index', False) @@ -771,7 +791,7 @@ def add_column(self, **kwargs): # noqa: C901 "Please ensure the new column complies with the spec. " "This will raise an error in a future version of HDMF." % (name, self.__class__.__name__, spec_index)) - warn(msg) + warn(msg, stacklevel=2) spec_col_cls = self.__uninit_cols[name].get('class', VectorData) if col_cls != spec_col_cls: @@ -780,7 +800,7 @@ def add_column(self, **kwargs): # noqa: C901 "Please ensure the new column complies with the spec. " "This will raise an error in a future version of HDMF." % (name, self.__class__.__name__, spec_col_cls)) - warn(msg) + warn(msg, stacklevel=2) ckwargs = dict(kwargs) @@ -1186,6 +1206,35 @@ def to_dataframe(self, **kwargs): ret = self.__get_selection_as_df(sel) return ret + def _repr_html_(self) -> str: + """Generates the HTML representation of the object.""" + header_text = self.name if self.name == self.__class__.__name__ else f"{self.name} ({self.__class__.__name__})" + html_repr = self.css_style + self.js_script + html_repr += "
" + html_repr += f"

{header_text}

" + html_repr += self.generate_html_repr() + html_repr += "
" + return html_repr + + def generate_html_repr(self, level: int = 0, access_code: str = "", nrows: int = 4): + out = "" + for key, value in self.fields.items(): + if key not in ("id", "colnames", "columns"): + out += self._generate_field_html(key, value, level, access_code) + + inside = f"{self[:min(nrows, len(self))].to_html()}" + + if len(self) == nrows + 1: + inside += "

... and 1 more row.

" + elif len(self) > nrows + 1: + inside += f"

... and {len(self) - nrows} more rows.

" + + out += ( + f'
table{inside}
' + ) + return out + @classmethod @docval( {'name': 'df', 'type': pd.DataFrame, 'doc': 'source DataFrame'}, @@ -1468,7 +1517,7 @@ def _validate_on_set_parent(self): if set(table_ancestor_ids).isdisjoint(self_ancestor_ids): msg = (f"The linked table for DynamicTableRegion '{self.name}' does not share an ancestor with the " "DynamicTableRegion.") - warn(msg) + warn(msg, stacklevel=2) return super()._validate_on_set_parent() diff --git a/src/hdmf/container.py b/src/hdmf/container.py index c83f85e1c..229e20083 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -2,6 +2,7 @@ from abc import abstractmethod from collections import OrderedDict from copy import deepcopy +from typing import Type from uuid import uuid4 from warnings import warn @@ -562,8 +563,10 @@ def __repr__(self): template += " {}: {}\n".format(k, v) return template - def _repr_html_(self): - CSS_STYLE = """ + @property + def css_style(self) -> str: + """CSS styles for the HTML representation.""" + return """ """ - JS_SCRIPT = """ + @property + def js_script(self) -> str: + """JavaScript for the HTML representation.""" + return """ """ - if self.name == self.__class__.__name__: - header_text = self.name - else: - header_text = f"{self.name} ({self.__class__.__name__})" - html_repr = CSS_STYLE - html_repr += JS_SCRIPT + + def _repr_html_(self) -> str: + """Generates the HTML representation of the object.""" + header_text = self.name if self.name == self.__class__.__name__ else f"{self.name} ({self.__class__.__name__})" + html_repr = self.css_style + self.js_script html_repr += "
" - html_repr += ( - f"

{header_text}

" - ) - html_repr += self._generate_html_repr(self.fields) + html_repr += f"

{header_text}

" + html_repr += self._generate_html_repr(self.fields, is_field=True) html_repr += "
" return html_repr - def _generate_html_repr(self, fields, level=0, access_code=".fields"): + def _generate_html_repr(self, fields, level=0, access_code="", is_field=False): + """Recursively generates HTML representation for fields.""" html_repr = "" if isinstance(fields, dict): for key, value in fields.items(): - current_access_code = f"{access_code}['{key}']" - if ( - isinstance(value, (list, dict, np.ndarray)) - or hasattr(value, "fields") - ): - label = key - if isinstance(value, dict): - label += f" ({len(value)})" - - html_repr += ( - f'
{label}' - ) - if hasattr(value, "fields"): - value = value.fields - current_access_code = current_access_code + ".fields" - html_repr += self._generate_html_repr( - value, level + 1, current_access_code - ) - html_repr += "
" - else: - html_repr += ( - f'
{key}: {value}
' - ) + current_access_code = f"{access_code}.{key}" if is_field else f"{access_code}['{key}']" + html_repr += self._generate_field_html(key, value, level, current_access_code) elif isinstance(fields, list): for index, item in enumerate(fields): - current_access_code = f"{access_code}[{index}]" - html_repr += ( - f'
{str(item)}
' - ) + access_code += f'[{index}]' + html_repr += self._generate_field_html(index, item, level, access_code) elif isinstance(fields, np.ndarray): - str_ = str(fields).replace("\n", "
") - html_repr += ( - f'
{str_}
' - ) + html_repr += self._generate_array_html(fields, level) else: pass return html_repr + def _generate_field_html(self, key, value, level, access_code): + """Generates HTML for a single field.""" + + if isinstance(value, (int, float, str, bool)): + return f'
{key}: {value}
' + + if hasattr(value, "generate_html_repr"): + html_content = value.generate_html_repr(level + 1, access_code) + + elif hasattr(value, '__repr_html__'): + html_content = value.__repr_html__() + + elif hasattr(value, "fields"): + html_content = self._generate_html_repr(value.fields, level + 1, access_code, is_field=True) + elif isinstance(value, (list, dict, np.ndarray)): + html_content = self._generate_html_repr(value, level + 1, access_code, is_field=False) + else: + html_content = f'{value}' + html_repr = ( + f'
{key}' + ) + html_repr += html_content + html_repr += "
" + + return html_repr + + def _generate_array_html(self, array, level): + """Generates HTML for a NumPy array.""" + str_ = str(array).replace("\n", "
") + return f'
{str_}
' + @staticmethod def __smart_str(v, num_indent): """ @@ -739,11 +747,34 @@ def __smart_str_dict(d, num_indent): out += '\n' + indent + right_br return out - def set_data_io(self, dataset_name, data_io_class, **kwargs): + def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kwargs: dict = None, **kwargs): + """ + Apply DataIO object to a dataset field of the Container. + + Parameters + ---------- + dataset_name: str + Name of dataset to wrap in DataIO + data_io_class: Type[DataIO] + Class to use for DataIO, e.g. H5DataIO or ZarrDataIO + data_io_kwargs: dict + keyword arguments passed to the constructor of the DataIO class. + **kwargs: + DEPRECATED. Use data_io_kwargs instead. + kwargs are passed to the constructor of the DataIO class. + """ + if kwargs or (data_io_kwargs is None): + warn( + "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a " + "dictionary to the `data_io_kwargs` parameter instead.", + DeprecationWarning, + stacklevel=2 + ) + data_io_kwargs = kwargs data = self.fields.get(dataset_name) if data is None: raise ValueError(f"{dataset_name} is None and cannot be wrapped in a DataIO class") - self.fields[dataset_name] = data_io_class(data=data, **kwargs) + self.fields[dataset_name] = data_io_class(data=data, **data_io_kwargs) class Data(AbstractContainer): @@ -756,6 +787,8 @@ class Data(AbstractContainer): def __init__(self, **kwargs): data = popargs('data', kwargs) super().__init__(**kwargs) + + self._validate_new_data(data) self.__data = data @property @@ -776,10 +809,28 @@ def set_dataio(self, **kwargs): """ Apply DataIO object to the data held by this Data object """ + warn( + "Data.set_dataio() is deprecated. Please use Data.set_data_io() instead.", + DeprecationWarning, + stacklevel=2, + ) dataio = getargs('dataio', kwargs) dataio.data = self.__data self.__data = dataio + def set_data_io(self, data_io_class: Type[DataIO], data_io_kwargs: dict) -> None: + """ + Apply DataIO object to the data held by this Data object. + + Parameters + ---------- + data_io_class: Type[DataIO] + The DataIO to apply to the data held by this Data. + data_io_kwargs: dict + The keyword arguments to pass to the DataIO. + """ + self.__data = data_io_class(data=self.__data, **data_io_kwargs) + @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'}) def transform(self, **kwargs): """ @@ -815,6 +866,7 @@ def get(self, args): return self.data[args] def append(self, arg): + self._validate_new_data_element(arg) self.__data = append_data(self.__data, arg) def extend(self, arg): @@ -824,8 +876,23 @@ def extend(self, arg): :param arg: The iterable to add to the end of this VectorData """ + self._validate_new_data(arg) self.__data = extend_data(self.__data, arg) + def _validate_new_data(self, data): + """Function to validate a new array that will be set or added to data. Raises an error if the data is invalid. + + Subclasses should override this function to perform class-specific validation. + """ + pass + + def _validate_new_data_element(self, arg): + """Function to validate a new value that will be added to the data. Raises an error if the data is invalid. + + Subclasses should override this function to perform class-specific validation. + """ + pass + class DataRegion(Data): diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 3781abe8e..f1eee655f 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -290,7 +290,7 @@ def __init__(self, **kwargs): dict( name="chunk_mb", type=(float, int), - doc="Size of the HDF5 chunk in megabytes. Recommended to be less than 1MB.", + doc="Size of the HDF5 chunk in megabytes.", default=None, ) ) @@ -1061,6 +1061,8 @@ def __len__(self): return self.__shape[0] if not self.valid: raise InvalidDataIOError("Cannot get length of data. Data is not valid.") + if isinstance(self.data, AbstractDataChunkIterator): + return self.data.maxshape[0] return len(self.data) def __bool__(self): diff --git a/src/hdmf/spec/namespace.py b/src/hdmf/spec/namespace.py index 73c41a1d8..a2ae0bd37 100644 --- a/src/hdmf/spec/namespace.py +++ b/src/hdmf/spec/namespace.py @@ -50,13 +50,13 @@ def __init__(self, **kwargs): self['full_name'] = full_name if version == str(SpecNamespace.UNVERSIONED): # the unversioned version may be written to file as a string and read from file as a string - warn("Loaded namespace '%s' is unversioned. Please notify the extension author." % name) + warn("Loaded namespace '%s' is unversioned. Please notify the extension author." % name, stacklevel=2) version = SpecNamespace.UNVERSIONED if version is None: # version is required on write -- see YAMLSpecWriter.write_namespace -- but can be None on read in order to # be able to read older files with extensions that are missing the version key. warn(("Loaded namespace '%s' is missing the required key 'version'. Version will be set to '%s'. " - "Please notify the extension author.") % (name, SpecNamespace.UNVERSIONED)) + "Please notify the extension author.") % (name, SpecNamespace.UNVERSIONED), stacklevel=2) version = SpecNamespace.UNVERSIONED self['version'] = version if date is not None: @@ -529,7 +529,7 @@ def load_namespaces(self, **kwargs): if ns['version'] != self.__namespaces.get(ns['name'])['version']: # warn if the cached namespace differs from the already loaded namespace warn("Ignoring cached namespace '%s' version %s because version %s is already loaded." - % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version'])) + % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']), stacklevel=2) else: to_load.append(ns) # now load specs into namespace diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index f383fd34a..9a9d876c3 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -318,7 +318,7 @@ def __init__(self, **kwargs): default_name = getargs('default_name', kwargs) if default_name: if name is not None: - warn("found 'default_name' with 'name' - ignoring 'default_name'") + warn("found 'default_name' with 'name' - ignoring 'default_name'", stacklevel=2) else: self['default_name'] = default_name self.__attributes = dict() @@ -816,11 +816,6 @@ def data_type_inc(self): ''' The data type of target specification ''' return self.get(_target_type_key) - @property - def data_type(self): - ''' The data type of target specification ''' - return self.get(_target_type_key) - def is_many(self): return self.quantity not in (1, ZERO_OR_ONE) diff --git a/src/hdmf/spec/write.py b/src/hdmf/spec/write.py index 352e883f5..799ffb88a 100644 --- a/src/hdmf/spec/write.py +++ b/src/hdmf/spec/write.py @@ -247,7 +247,7 @@ def export_spec(ns_builder, new_data_types, output_dir): """ if len(new_data_types) == 0: - warnings.warn('No data types specified. Exiting.') + warnings.warn('No data types specified. Exiting.', stacklevel=2) return ns_path = ns_builder.name + '.namespace.yaml' diff --git a/src/hdmf/testing/validate_spec.py b/src/hdmf/testing/validate_spec.py index 89b8704bf..4d25ff084 100755 --- a/src/hdmf/testing/validate_spec.py +++ b/src/hdmf/testing/validate_spec.py @@ -32,7 +32,8 @@ def __init__(self): new_resolver = FixResolver() f_nwb = open(fpath_spec, 'r') - instance = yaml.safe_load(f_nwb) + yaml_obj = yaml.YAML(typ='safe', pure=True) + instance = yaml_obj.load(f_nwb) jsonschema.validate(instance, schema, resolver=new_resolver) diff --git a/src/hdmf/utils.py b/src/hdmf/utils.py index d85eb5c8c..b3c8129b7 100644 --- a/src/hdmf/utils.py +++ b/src/hdmf/utils.py @@ -67,7 +67,7 @@ def get_docval_macro(key=None): return tuple(__macros[key]) -def __type_okay(value, argtype, allow_none=False): +def check_type(value, argtype, allow_none=False): """Check a value against a type The difference between this function and :py:func:`isinstance` is that @@ -87,7 +87,7 @@ def __type_okay(value, argtype, allow_none=False): return allow_none if isinstance(argtype, str): if argtype in __macros: - return __type_okay(value, __macros[argtype], allow_none=allow_none) + return check_type(value, __macros[argtype], allow_none=allow_none) elif argtype == 'uint': return __is_uint(value) elif argtype == 'int': @@ -106,7 +106,7 @@ def __type_okay(value, argtype, allow_none=False): return __is_bool(value) return isinstance(value, argtype) elif isinstance(argtype, tuple) or isinstance(argtype, list): - return any(__type_okay(value, i) for i in argtype) + return any(check_type(value, i) for i in argtype) else: # argtype is None return True @@ -279,7 +279,7 @@ def __parse_args(validator, args, kwargs, enforce_type=True, enforce_shape=True, # we can use this to unwrap the dataset/attribute to use the "item" for docval to validate the type. argval = argval.value if enforce_type: - if not __type_okay(argval, arg['type']): + if not check_type(argval, arg['type']): if argval is None: fmt_val = (argname, __format_type(arg['type'])) type_errors.append("None is not allowed for '%s' (expected '%s', not None)" % fmt_val) @@ -336,7 +336,7 @@ def __parse_args(validator, args, kwargs, enforce_type=True, enforce_shape=True, # we can use this to unwrap the dataset/attribute to use the "item" for docval to validate the type. argval = argval.value if enforce_type: - if not __type_okay(argval, arg['type'], arg['default'] is None or arg.get('allow_none', False)): + if not check_type(argval, arg['type'], arg['default'] is None or arg.get('allow_none', False)): if argval is None and arg['default'] is None: fmt_val = (argname, __format_type(arg['type'])) type_errors.append("None is not allowed for '%s' (expected '%s', not None)" % fmt_val) @@ -434,7 +434,7 @@ def fmt_docval_args(func, kwargs): "removes all arguments not accepted by the function's docval, so if you are passing kwargs that " "includes extra arguments and the function's docval does not allow extra arguments (allow_extra=True " "is set), then you will need to pop the extra arguments out of kwargs before calling the function.", - PendingDeprecationWarning) + PendingDeprecationWarning, stacklevel=2) func_docval = getattr(func, docval_attr_name, None) ret_args = list() ret_kwargs = dict() @@ -488,7 +488,7 @@ def call_docval_func(func, kwargs): "removes all arguments not accepted by the function's docval, so if you are passing kwargs that " "includes extra arguments and the function's docval does not allow extra arguments (allow_extra=True " "is set), then you will need to pop the extra arguments out of kwargs before calling the function.", - PendingDeprecationWarning) + PendingDeprecationWarning, stacklevel=2) with warnings.catch_warnings(record=True): # catch and ignore only PendingDeprecationWarnings from fmt_docval_args so that two # PendingDeprecationWarnings saying the same thing are not raised @@ -613,7 +613,7 @@ def dec(func): msg = 'docval for {}: enum checking cannot be used with arg type {}'.format(a['name'], a['type']) raise Exception(msg) # check that enum allowed values are allowed by arg type - if any([not __type_okay(x, a['type']) for x in a['enum']]): + if any([not check_type(x, a['type']) for x in a['enum']]): msg = ('docval for {}: enum values are of types not allowed by arg type (got {}, ' 'expected {})'.format(a['name'], [type(x) for x in a['enum']], a['type'])) raise Exception(msg) @@ -645,7 +645,7 @@ def _check_args(args, kwargs): parse_warnings = parsed.get('future_warnings') if parse_warnings: msg = '%s: %s' % (func.__qualname__, ', '.join(parse_warnings)) - warnings.warn(msg, FutureWarning) + warnings.warn(msg, category=FutureWarning, stacklevel=3) for error_type, ExceptionType in (('type_errors', TypeError), ('value_errors', ValueError), @@ -837,6 +837,10 @@ class ExtenderMeta(ABCMeta): @classmethod def pre_init(cls, func): + """ + A decorator that sets a '__preinit' attribute on the target function and + then returns the function as a classmethod. + """ setattr(func, cls.__preinit, True) return classmethod(func) diff --git a/src/hdmf/validate/validator.py b/src/hdmf/validate/validator.py index 0104978ca..a6198ef8e 100644 --- a/src/hdmf/validate/validator.py +++ b/src/hdmf/validate/validator.py @@ -42,7 +42,7 @@ __allowable['numeric'] = set(chain.from_iterable(__allowable[k] for k in __allowable if 'int' in k or 'float' in k)) -def check_type(expected, received): +def check_type(expected, received, string_format=None): ''' *expected* should come from the spec *received* should come from the data @@ -52,6 +52,12 @@ def check_type(expected, received): raise ValueError('compound type shorter than expected') for i, exp in enumerate(DtypeHelper.simplify_cpd_type(expected)): rec = received[i] + if exp == "isodatetime": # short circuit for isodatetime + sub_string_format = string_format[i] + return ( + rec in __allowable[exp] or + rec in ("utf", "ascii") and sub_string_format == "isodatetime" + ) if rec not in __allowable[exp]: return False return True @@ -71,6 +77,11 @@ def check_type(expected, received): received = received.name elif isinstance(received, type): received = received.__name__ + if expected == "isodatetime": # short circuit for isodatetime + return ( + received in __allowable[expected] or + (received in ("utf", "ascii") and string_format == "isodatetime") + ) if isinstance(expected, RefSpec): expected = expected.reftype elif isinstance(expected, type): @@ -89,55 +100,64 @@ def get_iso8601_regex(): _iso_re = get_iso8601_regex() -def _check_isodatetime(s, default=None): +def get_string_format(data): + """Return the string format of the given data. Possible outputs are "isodatetime" and None. + """ + assert isinstance(data, (str, bytes)) try: - if _iso_re.match(pystr(s)) is not None: + if _iso_re.match(pystr(data)) is not None: return 'isodatetime' except Exception: pass - return default + return None class EmptyArrayError(Exception): pass -def get_type(data): +def get_type(data, builder_dtype=None): + """Return a tuple of (the string representation of the type, the format of the string data) for the given data.""" # String data if isinstance(data, str): - return _check_isodatetime(data, 'utf') + return 'utf', get_string_format(data) # Bytes data elif isinstance(data, bytes): - return _check_isodatetime(data, 'ascii') + return 'ascii', get_string_format(data) # RegionBuilder data elif isinstance(data, RegionBuilder): - return 'region' + return 'region', None # ReferenceBuilder data elif isinstance(data, ReferenceBuilder): - return 'object' + return 'object', None # ReferenceResolver data elif isinstance(data, ReferenceResolver): - return data.dtype + return data.dtype, None # Numpy nd-array data elif isinstance(data, np.ndarray): if data.size > 0: - return get_type(data[0]) + return get_type(data[0], builder_dtype) else: raise EmptyArrayError() # Numpy bool data elif isinstance(data, np.bool_): - return 'bool' + return 'bool', None if not hasattr(data, '__len__'): - return type(data).__name__ + return type(data).__name__, None # Case for h5py.Dataset and other I/O specific array types else: + # Compound dtype + if builder_dtype and isinstance(builder_dtype, list): + dtypes = [] + string_formats = [] + for i in range(len(builder_dtype)): + dtype, string_format = get_type(data[0][i]) + dtypes.append(dtype) + string_formats.append(string_format) + return dtypes, string_formats # Object has 'dtype' attribute, e.g., an h5py.Dataset if hasattr(data, 'dtype'): - # Compound data type - if isinstance(data.dtype, list): - return [get_type(data[0][i]) for i in range(len(data.dtype))] - # Variable length data type - elif data.dtype.metadata is not None and data.dtype.metadata.get('vlen') is not None: + if data.dtype.metadata is not None and data.dtype.metadata.get('vlen') is not None: # Try to determine dtype from the first array element if len(data) > 0: return get_type(data[0]) @@ -145,16 +165,16 @@ def get_type(data): else: # Empty string array if data.dtype.metadata["vlen"] == str: - return "utf" + return "utf", get_string_format(data) # Undetermined variable length data type. else: # pragma: no cover raise EmptyArrayError() # pragma: no cover - # Standard date type (i.e, not compound or vlen. + # Standard data type (i.e., not compound or vlen) else: - return data.dtype + return data.dtype, None # If all else has failed, try to determine the datatype from the first element of the array if len(data) > 0: - return get_type(data[0]) + return get_type(data[0], builder_dtype) else: raise EmptyArrayError() @@ -336,7 +356,7 @@ def validate(self, **kwargs): if not isinstance(value, BaseBuilder): expected = '%s reference' % spec.dtype.reftype try: - value_type = get_type(value) + value_type, _ = get_type(value) ret.append(DtypeError(self.get_spec_loc(spec), expected, value_type)) except EmptyArrayError: # do not validate dtype of empty array. HDMF does not yet set dtype when writing a list/tuple @@ -349,8 +369,8 @@ def validate(self, **kwargs): ret.append(IncorrectDataType(self.get_spec_loc(spec), spec.dtype.target_type, data_type)) else: try: - dtype = get_type(value) - if not check_type(spec.dtype, dtype): + dtype, string_format = get_type(value) + if not check_type(spec.dtype, dtype, string_format): ret.append(DtypeError(self.get_spec_loc(spec), spec.dtype, dtype)) except EmptyArrayError: # do not validate dtype of empty array. HDMF does not yet set dtype when writing a list/tuple @@ -411,14 +431,17 @@ def validate(self, **kwargs): data = builder.data if self.spec.dtype is not None: try: - dtype = get_type(data) - if not check_type(self.spec.dtype, dtype): + dtype, string_format = get_type(data, builder.dtype) + if not check_type(self.spec.dtype, dtype, string_format): ret.append(DtypeError(self.get_spec_loc(self.spec), self.spec.dtype, dtype, location=self.get_builder_loc(builder))) except EmptyArrayError: # do not validate dtype of empty array. HDMF does not yet set dtype when writing a list/tuple pass - shape = get_data_shape(data) + if isinstance(builder.dtype, list): + shape = (len(builder.data), ) # only 1D datasets with compound types are supported + else: + shape = get_data_shape(data) if not check_shape(self.spec.shape, shape): if shape is None: ret.append(ExpectedArrayError(self.get_spec_loc(self.spec), self.spec.shape, str(data), diff --git a/test_gallery.py b/test_gallery.py index 970ef93f1..c3128b8fd 100644 --- a/test_gallery.py +++ b/test_gallery.py @@ -26,16 +26,11 @@ def _import_from_file(script): _numpy_warning_re = "numpy.ufunc size changed, may indicate binary incompatibility. Expected 216, got 192" -_distutils_warning_re = "distutils Version classes are deprecated. Use packaging.version instead." - _experimental_warning_re = ( "[a-zA-Z0-9]+ is experimental -- it may be removed in the future " "and is not guaranteed to maintain backward compatibility" ) -pydantic_warning_re = ("Support for class-based `config` is deprecated, use ConfigDict instead.") - - def run_gallery_tests(): global TOTAL, FAILURES, ERRORS logging.info("Testing execution of Sphinx Gallery files") @@ -48,6 +43,10 @@ def run_gallery_tests(): gallery_file_names.append(os.path.join(root, f)) warnings.simplefilter("error") + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, # these can be triggered by downstream packages. ignore for these tests + ) TOTAL += len(gallery_file_names) for script in gallery_file_names: @@ -59,12 +58,6 @@ def run_gallery_tests(): message=_experimental_warning_re, category=UserWarning, ) - warnings.filterwarnings( - # this warning is triggered from pandas when HDMF is installed with the minimum requirements - "ignore", - message=_distutils_warning_re, - category=DeprecationWarning, - ) warnings.filterwarnings( # this warning is triggered when some numpy extension code in an upstream package was compiled # against a different version of numpy than the one installed @@ -72,17 +65,10 @@ def run_gallery_tests(): message=_numpy_warning_re, category=RuntimeWarning, ) - warnings.filterwarnings( - # this warning is triggered when some linkml dependency like curies uses pydantic in a way that - # will be deprecated in the future - "ignore", - message=pydantic_warning_re, - category=DeprecationWarning, - ) _import_from_file(script) except (ImportError, ValueError) as e: - if "linkml" in str(e) and sys.version_info < (3, 9): - pass # this is OK because plot_term_set.py and plot_external_resources.py cannot be run on Python 3.8 + if "linkml" in str(e): + pass # this is OK because linkml is not always installed else: raise e except Exception: diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 5635b12d1..0c117820b 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -497,7 +497,7 @@ def setUp(self): ], links=[ LinkSpec( - doc='A composition inside with a fixed name', + doc='A composition inside without a fixed name', name="my_baz1_link", target_type='Baz1' ), @@ -517,7 +517,7 @@ def test_gen_parent_class(self): {'name': 'name', 'type': str, 'doc': 'the name of this container'}, {'name': 'my_baz1', 'doc': 'A composition inside with a fixed name', 'type': baz1_cls}, {'name': 'my_baz2', 'doc': 'A composition inside with a fixed name', 'type': baz2_cls}, - {'name': 'my_baz1_link', 'doc': 'A composition inside with a fixed name', 'type': baz1_cls}, + {'name': 'my_baz1_link', 'doc': 'A composition inside without a fixed name', 'type': baz1_cls}, )) def test_init_fields(self): @@ -537,8 +537,7 @@ def test_init_fields(self): }, { 'name': 'my_baz1_link', - 'doc': 'A composition inside with a fixed name', - 'required_name': 'my_baz1_link' + 'doc': 'A composition inside without a fixed name', }, )) @@ -548,7 +547,7 @@ def test_set_field(self): baz3_cls = self.type_map.get_dt_container_cls('Baz3', CORE_NAMESPACE) baz1 = baz1_cls(name="my_baz1") baz2 = baz2_cls(name="my_baz2") - baz1_link = baz1_cls(name="my_baz1_link") + baz1_link = baz1_cls(name="any_name") baz3 = baz3_cls(name="test", my_baz1=baz1, my_baz2=baz2, my_baz1_link=baz1_link) self.assertEqual(baz3.my_baz1, baz1) self.assertEqual(baz3.my_baz2, baz2) @@ -573,13 +572,6 @@ def test_set_field_bad(self): with self.assertRaisesWith(ValueError, msg): baz3_cls(name="test", my_baz1=baz1, my_baz2=baz2, my_baz1_link=baz1_link) - baz1 = baz1_cls(name="my_baz1") - baz2 = baz2_cls(name="my_baz2") - baz1_link = baz1_cls(name="test") - msg = "Field 'my_baz1_link' on Baz3 must be named 'my_baz1_link'." - with self.assertRaisesWith(ValueError, msg): - baz3_cls(name="test", my_baz1=baz1, my_baz2=baz2, my_baz1_link=baz1_link) - class TestGetClassSeparateNamespace(TestCase): @@ -1049,7 +1041,7 @@ def test_process_field_spec_link(self): spec=GroupSpec('dummy', 'doc') ) - expected = {'__fields__': [{'name': 'attr3', 'doc': 'a link', 'required_name': 'attr3'}]} + expected = {'__fields__': [{'name': 'attr3', 'doc': 'a link'}]} self.assertDictEqual(classdict, expected) def test_post_process_fixed_name(self): diff --git a/tests/unit/common/test_resources.py b/tests/unit/common/test_resources.py index 796f75db4..8cbd8291e 100644 --- a/tests/unit/common/test_resources.py +++ b/tests/unit/common/test_resources.py @@ -4,7 +4,7 @@ from hdmf import TermSet, TermSetWrapper from hdmf.common.resources import HERD, Key from hdmf import Data, Container, HERDManager -from hdmf.testing import TestCase, H5RoundTripMixin, remove_test_file +from hdmf.testing import TestCase, remove_test_file import numpy as np from tests.unit.build_tests.test_io_map import Bar from tests.unit.helpers.utils import create_test_type_map, CORE_NAMESPACE @@ -25,7 +25,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) -class TestHERD(H5RoundTripMixin, TestCase): +class TestHERD(TestCase): def setUpContainer(self): er = HERD() @@ -88,18 +88,18 @@ def test_to_dataframe(self): file_1 = HERDManagerContainer(name='file_1') file_2 = HERDManagerContainer(name='file_2') - k1, e1 = er.add_ref(file=file_1, - container=data1, - field='species', - key='Mus musculus', - entity_id='NCBI:txid10090', - entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=10090') - k2, e2 = er.add_ref(file=file_2, - container=data2, - field='species', - key='Homo sapiens', - entity_id='NCBI:txid9606', - entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=9606') + er.add_ref(file=file_1, + container=data1, + field='species', + key='Mus musculus', + entity_id='NCBI:txid10090', + entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=10090') + er.add_ref(file=file_2, + container=data2, + field='species', + key='Homo sapiens', + entity_id='NCBI:txid9606', + entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=9606') # Convert to dataframe and compare against the expected result result_df = er.to_dataframe() @@ -268,6 +268,21 @@ def test_add_ref_search_for_file_error(self): entity_id='entity_id1', entity_uri='entity1') + # TODO: Add this once you've created a HDMF_file to rework testing + # def test_add_ref_file_mismatch(self): + # file = HERDManagerContainer(name='file') + # file2 = HERDManagerContainer() + # + # nested_child = Container(name='nested_child') + # child = Container(name='child') + # nested_child.parent = child + # child.parent = file + # + # er = HERD() + # with self.assertRaises(ValueError): + # er.add_ref(file=file2, container=nested_child, key='key1', + # entity_id='entity_id1', entity_uri='entity1') + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") def test_check_termset_wrapper(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') @@ -289,7 +304,7 @@ def test_check_termset_wrapper(self): self.assertTrue(isinstance(ret[0][2], TermSetWrapper)) @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") - def test_add_ref_termset_data(self): + def test_add_ref_container_data(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') er = HERD() em = HERDManagerContainer() @@ -305,14 +320,14 @@ def test_add_ref_termset_data(self): species.parent = em - er.add_ref_term_set(root_container=em) + er.add_ref_container(root_container=em) self.assertEqual(er.keys.data, [('Homo sapiens',)]) self.assertEqual(er.entities.data, [('NCBI_TAXON:9606', 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606')]) self.assertEqual(er.objects.data, [(0, col1.object_id, 'VectorData', '', '')]) @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") - def test_add_ref_termset_attr(self): + def test_add_ref_container_attr(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') er = HERD() em = HERDManagerContainer() @@ -328,12 +343,159 @@ def test_add_ref_termset_attr(self): species.parent = em - er.add_ref_term_set(root_container=em) + er.add_ref_container(root_container=em) self.assertEqual(er.keys.data, [('Homo sapiens',)]) self.assertEqual(er.entities.data, [('NCBI_TAXON:9606', 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606')]) self.assertEqual(er.objects.data, [(0, col1.object_id, 'VectorData', 'description', '')]) + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + def test_add_ref_termset(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + er = HERD() + em = HERDManagerContainer() + + col1 = VectorData(name='Species_Data', + description='species from NCBI and Ensemble', + data=['Homo sapiens']) + + species = DynamicTable(name='species', description='My species', columns=[col1],) + + er.add_ref_termset(file=em, + container=species, + attribute='Species_Data', + key='Homo sapiens', + termset=terms + ) + self.assertEqual(er.keys.data, [('Homo sapiens',)]) + self.assertEqual(er.entities.data, [('NCBI_TAXON:9606', + 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606')]) + self.assertEqual(er.objects.data, [(0, col1.object_id, 'VectorData', '', '')]) + + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + def test_add_ref_termset_data_object_error(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + er = HERD() + + col1 = VectorData(name='Species_Data', + description='species from NCBI and Ensemble', + data=['Homo sapiens']) + + with self.assertRaises(ValueError): + er.add_ref_termset( + container=col1, + attribute='description', + termset=terms + ) + + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + def test_add_ref_termset_attribute_none(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + er = HERD() + em = HERDManagerContainer() + + col1 = VectorData(name='Species_Data', + description='species from NCBI and Ensemble', + data=['Homo sapiens']) + + species = DynamicTable(name='species', description='My species', columns=[col1],) + + er.add_ref_termset(file=em, + container=species['Species_Data'], + termset=terms + ) + self.assertEqual(er.keys.data, [('Homo sapiens',)]) + self.assertEqual(er.entities.data, [('NCBI_TAXON:9606', + 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606')]) + self.assertEqual(er.objects.data, [(0, col1.object_id, 'VectorData', '', '')]) + + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + def test_add_ref_termset_data_object_list(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + er = HERD() + em = HERDManagerContainer() + + col1 = VectorData(name='Homo sapiens', + description='species from NCBI and Ensemble', + data=['Homo sapiens']) + + species = DynamicTable(name='species', description='My species', columns=[col1],) + + er.add_ref_termset(file=em, + container=species, + attribute='colnames', + termset=terms + ) + self.assertEqual(er.keys.data, [('Homo sapiens',)]) + self.assertEqual(er.entities.data, [('NCBI_TAXON:9606', + 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606')]) + self.assertEqual(er.objects.data, [(0, species.object_id, 'DynamicTable', 'colnames', '')]) + + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + def test_add_ref_termset_bulk(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + er = HERD() + em = HERDManagerContainer() + + col1 = VectorData(name='Species_Data', + description='species from NCBI and Ensemble', + data=['Homo sapiens', 'Mus musculus']) + + species = DynamicTable(name='species', description='My species', columns=[col1],) + + er.add_ref_termset(file=em, + container=species, + attribute='Species_Data', + termset=terms + ) + self.assertEqual(er.keys.data, [('Homo sapiens',), ('Mus musculus',)]) + self.assertEqual(er.entities.data, [('NCBI_TAXON:9606', + 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606'), + ('NCBI_TAXON:10090', + 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=10090')]) + self.assertEqual(er.objects.data, [(0, col1.object_id, 'VectorData', '', '')]) + + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + def test_add_ref_termset_missing_terms(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + er = HERD() + em = HERDManagerContainer() + + col1 = VectorData(name='Species_Data', + description='species from NCBI and Ensemble', + data=['Homo sapiens', 'missing_term']) + + species = DynamicTable(name='species', description='My species', columns=[col1],) + + missing_terms = er.add_ref_termset(file=em, + container=species, + attribute='Species_Data', + termset=terms + ) + self.assertEqual(er.keys.data, [('Homo sapiens',)]) + self.assertEqual(er.entities.data, [('NCBI_TAXON:9606', + 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606')]) + self.assertEqual(er.objects.data, [(0, col1.object_id, 'VectorData', '', '')]) + self.assertEqual(missing_terms, {'missing_terms': ['missing_term']}) + + @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + def test_add_ref_termset_missing_file_error(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + er = HERD() + + col1 = VectorData(name='Species_Data', + description='species from NCBI and Ensemble', + data=['Homo sapiens']) + + species = DynamicTable(name='species', description='My species', columns=[col1],) + + with self.assertRaises(ValueError): + er.add_ref_termset( + container=species, + attribute='Species_Data', + termset=terms + ) + def test_get_file_from_container(self): file = HERDManagerContainer(name='file') container = Container(name='name') @@ -826,42 +988,21 @@ def test_object_key_existing_key_new_object(self): entity_uri='entity_uri2') self.assertEqual(er.object_keys.data, [(0, 0), (1, 0)]) - def test_object_key_existing_key_new_object_error(self): + def test_reuse_key_string(self): + # With the key and entity existing, the EntityKeyTable should not have duplicates er = HERD() - data_1 = Data(name='data_name', data=np.array([('Mus musculus', 9, 81.0), ('Homo sapien', 3, 27.0)], - dtype=[('species', 'U14'), ('age', 'i4'), ('weight', 'f4')])) + data_1 = Data(name='data_name', + data=np.array([('Mus musculus', 9, 81.0), ('Homo sapien', 3, 27.0), ('mouse', 3, 27.0)], + dtype=[('species', 'U14'), ('age', 'i4'), ('weight', 'f4')])) er.add_ref(file=HERDManagerContainer(name='file'), container=data_1, key='Mus musculus', entity_id='NCBI:txid10090', entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=10090') - key = er._add_key('key') - with self.assertRaises(ValueError): - er.add_ref(file=HERDManagerContainer(name='file'), - container=data_1, - key=key, - entity_id='entity1', - entity_uri='entity_uri1') - - def test_reuse_key_reuse_entity(self): - # With the key and entity existing, the EntityKeyTable should not have duplicates - er = HERD() - data_1 = Data(name='data_name', data=np.array([('Mus musculus', 9, 81.0), ('Homo sapien', 3, 27.0)], - dtype=[('species', 'U14'), ('age', 'i4'), ('weight', 'f4')])) - - data_2 = Data(name='data_name', data=np.array([('Mus musculus', 9, 81.0), ('Homo sapien', 3, 27.0)], - dtype=[('species', 'U14'), ('age', 'i4'), ('weight', 'f4')])) - er.add_ref(file=HERDManagerContainer(name='file'), container=data_1, key='Mus musculus', - entity_id='NCBI:txid10090', - entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=10090') - existing_key = er.get_key('Mus musculus') - er.add_ref(file=HERDManagerContainer(name='file'), - container=data_2, - key=existing_key, entity_id='NCBI:txid10090') self.assertEqual(er.entity_keys.data, [(0, 0)]) @@ -922,7 +1063,7 @@ def test_entity_uri_error(self): key='Mus musculus', entity_id='NCBI:txid10090') - def test_entity_uri_reuse_error(self): + def test_entity_uri_warning(self): er = HERD() data_1 = Data(name='data_name', data=np.array([('Mus musculus', 9, 81.0), ('Homo sapien', 3, 27.0)], dtype=[('species', 'U14'), ('age', 'i4'), ('weight', 'f4')])) @@ -936,7 +1077,7 @@ def test_entity_uri_reuse_error(self): entity_id='NCBI:txid10090', entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=10090') existing_key = er.get_key('Mus musculus') - with self.assertRaises(ValueError): + with self.assertWarns(Warning): er.add_ref(file=HERDManagerContainer(name='file'), container=data_2, key=existing_key, @@ -963,32 +1104,32 @@ def test_key_without_entity_error(self): def test_check_object_field_add(self): er = HERD() data = Data(name="species", data=['Homo sapiens', 'Mus musculus']) - er._check_object_field(file=HERDManagerContainer(name='file'), + file = HERDManagerContainer(name='file') + _dict = er._check_object_field(file=file, container=data, relative_path='', field='') - - self.assertEqual(er.objects.data, [(0, data.object_id, 'Data', '', '')]) + expected = {'file_object_id': file.object_id, + 'files_idx': None, + 'container': data, + 'relative_path': '', + 'field': ''} + self.assertEqual(_dict, expected) def test_check_object_field_multi_files(self): er = HERD() data = Data(name="species", data=['Homo sapiens', 'Mus musculus']) file = HERDManagerContainer(name='file') - - er._check_object_field(file=file, container=data, relative_path='', field='') + er._add_file(file.object_id) er._add_file(file.object_id) - data2 = Data(name="species", data=['Homo sapiens', 'Mus musculus']) with self.assertRaises(ValueError): - er._check_object_field(file=file, container=data2, relative_path='', field='') + er._check_object_field(file=file, container=data, relative_path='', field='') def test_check_object_field_multi_error(self): er = HERD() data = Data(name="species", data=['Homo sapiens', 'Mus musculus']) - er._check_object_field(file=HERDManagerContainer(name='file'), - container=data, - relative_path='', - field='') + er._add_object(files_idx=0, container=data, relative_path='', field='') er._add_object(files_idx=0, container=data, relative_path='', field='') with self.assertRaises(ValueError): er._check_object_field(file=HERDManagerContainer(name='file'), @@ -1063,14 +1204,6 @@ def test_add_ref_compound_data(self): self.assertEqual(er.entities.data, [('NCBI:txid10090', 'entity_0_uri')]) self.assertEqual(er.objects.data, [(0, data.object_id, 'Data', '', 'species')]) - def test_roundtrip(self): - read_container = self.roundtripContainer() - pd.testing.assert_frame_equal(read_container.to_dataframe(), self.container.to_dataframe()) - - def test_roundtrip_export(self): - read_container = self.roundtripExportContainer() - pd.testing.assert_frame_equal(read_container.to_dataframe(), self.container.to_dataframe()) - class TestHERDNestedAttributes(TestCase): @@ -1133,7 +1266,7 @@ class TestHERDGetKey(TestCase): def setUp(self): self.er = HERD() - def test_get_key_error_more_info(self): + def test_get_key_multiple(self): self.er.add_ref(file=HERDManagerContainer(name='file'), container=Container(name='Container'), key='key1', @@ -1145,9 +1278,11 @@ def test_get_key_error_more_info(self): entity_id="id12", entity_uri='url21') - msg = "There are more than one key with that name. Please search with additional information." - with self.assertRaisesWith(ValueError, msg): - _ = self.er.get_key(key_name='key1') + keys = self.er.get_key(key_name='key1') + self.assertIsInstance(keys[0], Key) + self.assertIsInstance(keys[1], Key) + self.assertEqual(keys[0].idx, 0) + self.assertEqual(keys[1].idx, 1) def test_get_key(self): self.er.add_ref(file=HERDManagerContainer(name='file'), diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index 88f8ca07b..7246a8ba8 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -9,13 +9,26 @@ from hdmf import TermSet, TermSetWrapper from hdmf.backends.hdf5 import H5DataIO, HDF5IO from hdmf.backends.hdf5.h5tools import H5_TEXT, H5PY_3 -from hdmf.common import (DynamicTable, VectorData, VectorIndex, ElementIdentifiers, EnumData, - DynamicTableRegion, get_manager, SimpleMultiContainer) +from hdmf.common import ( + DynamicTable, + VectorData, + VectorIndex, + ElementIdentifiers, + EnumData, + DynamicTableRegion, + get_manager, + SimpleMultiContainer, +) from hdmf.testing import TestCase, H5RoundTripMixin, remove_test_file from hdmf.utils import StrDataset from hdmf.data_utils import DataChunkIterator -from tests.unit.helpers.utils import get_temp_filepath +from tests.unit.helpers.utils import ( + get_temp_filepath, + FooExtendDynamicTable0, + FooExtendDynamicTable1, + FooExtendDynamicTable2, +) try: import linkml_runtime # noqa: F401 @@ -565,9 +578,9 @@ def test_getitem_row_slice_with_step(self): rows = table[0:5:2] self.assertIsInstance(rows, pd.DataFrame) self.assertTupleEqual(rows.shape, (3, 3)) - self.assertEqual(rows.iloc[2][0], 5) - self.assertEqual(rows.iloc[2][1], 50.0) - self.assertEqual(rows.iloc[2][2], 'lizard') + self.assertEqual(rows.iloc[2].iloc[0], 5) + self.assertEqual(rows.iloc[2].iloc[1], 50.0) + self.assertEqual(rows.iloc[2].iloc[2], 'lizard') def test_getitem_invalid_keytype(self): table = self.with_spec() @@ -771,6 +784,36 @@ def test_repr(self): expected = expected % id(table) self.assertEqual(str(table), expected) + def test_repr_html(self): + table = self.with_spec() + html = table._repr_html_() + + assert html == ( + '\n \n \n ' + '\n

with_spec (DynamicTable)

description: ' + 'a test table
table\n \n \n \n ' + '\n \n \n \n \n \n ' + '\n \n \n \n \n \n ' + '\n
foobarbaz
id
' + ) + + def test_add_column_existing_attr(self): table = self.with_table_columns() attrs = ['name', 'description', 'parent', 'id', 'fields'] # just a few @@ -1349,6 +1392,38 @@ def test_identifier_search_with_bad_ids(self): _ = (self.e == 'test') +class TestBadElementIdentifiers(TestCase): + + def test_bad_dtype(self): + with self.assertRaisesWith(ValueError, "ElementIdentifiers must contain integers"): + ElementIdentifiers(name='ids', data=["1", "2"]) + + with self.assertRaisesWith(ValueError, "ElementIdentifiers must contain integers"): + ElementIdentifiers(name='ids', data=np.array(["1", "2"])) + + with self.assertRaisesWith(ValueError, "ElementIdentifiers must contain integers"): + ElementIdentifiers(name='ids', data=[1.0, 2.0]) + + def test_dci_int_ok(self): + a = np.arange(30) + dci = DataChunkIterator(data=a, buffer_size=1) + e = ElementIdentifiers(name='ids', data=dci) # test that no error is raised + self.assertIs(e.data, dci) + + def test_dci_float_bad(self): + a = np.arange(30.0) + dci = DataChunkIterator(data=a, buffer_size=1) + with self.assertRaisesWith(ValueError, "ElementIdentifiers must contain integers"): + ElementIdentifiers(name='ids', data=dci) + + def test_dataio_dci_ok(self): + a = np.arange(30) + dci = DataChunkIterator(data=a, buffer_size=1) + dio = H5DataIO(dci) + e = ElementIdentifiers(name='ids', data=dio) # test that no error is raised + self.assertIs(e.data, dio) + + class SubTable(DynamicTable): __columns__ = ( @@ -2676,3 +2751,19 @@ def test_list_prev_data_inc_precision_2steps(self): index.add_vector(list(range(65536 - 255))) self.assertEqual(index.data[0], 255) # make sure the 255 is upgraded self.assertEqual(type(index.data[0]), np.uint32) + + +class TestDynamicTableSubclassColumns(TestCase): + def setUp(self): + self.foo1 = FooExtendDynamicTable0() + self.foo2 = FooExtendDynamicTable1() + self.foo3 = FooExtendDynamicTable2() + + def test_columns(self): + self.assertEqual(self.foo1.__columns__, + ({'name': 'col1', 'description': '...'}, {'name': 'col2', 'description': '...'})) + self.assertEqual(self.foo2.__columns__, + ({'name': 'col1', 'description': '...'}, {'name': 'col2', 'description': '...'}, + {'name': 'col3', 'description': '...'}, {'name': 'col4', 'description': '...'}) +) + self.assertEqual(self.foo2.__columns__, self.foo3.__columns__) diff --git a/tests/unit/helpers/utils.py b/tests/unit/helpers/utils.py index 5d4bf16ec..0f4b3c4bf 100644 --- a/tests/unit/helpers/utils.py +++ b/tests/unit/helpers/utils.py @@ -3,7 +3,8 @@ from copy import copy, deepcopy from hdmf.build import BuildManager, ObjectMapper, TypeMap -from hdmf.container import Container, HERDManager, Data +from hdmf.common.table import DynamicTable +from hdmf.container import Container, HERDManager, Data, MultiContainerInterface from hdmf.spec import ( AttributeSpec, DatasetSpec, @@ -653,3 +654,50 @@ class CustomSpecNamespace(SpecNamespace): @classmethod def types_key(cls): return cls.__types_key + +class FooExtendDynamicTable0(DynamicTable): + """ + Within PyNWB, PlaneSegmentation extends DynamicTable and sets __columns__. This class is a helper + class for testing and is directly meant to test __gather_columns, i.e., class generation, downstream. + """ + __columns__ = ( + {'name': 'col1', 'description': '...'}, + {'name': 'col2', 'description': '...'}, + ) + + def __init__(self, **kwargs): + kwargs['name'] = 'foo0' + kwargs['description'] = '...' + super().__init__(**kwargs) + + +class FooExtendDynamicTable1(FooExtendDynamicTable0): + """ + In extensions, users can create new classes that inherit from classes that inherit from DynamicTable. + This is a helper class for testing and is directly meant to test __gather_columns, i.e., + class generation, downstream. + """ + __columns__ = ( + {'name': 'col3', 'description': '...'}, + {'name': 'col4', 'description': '...'}, + ) + + def __init__(self, **kwargs): + kwargs['name'] = 'foo1' + kwargs['description'] = '...' + super().__init__(**kwargs) + + +class FooExtendDynamicTable2(FooExtendDynamicTable1, MultiContainerInterface): + __clsconf__ = { + 'add': '...', + 'get': '...', + 'create': '...', + 'type': Container, + 'attr': '...' + } + + def __init__(self, **kwargs): + kwargs['name'] = 'foo2' + kwargs['description'] = '...' + super().__init__(**kwargs) diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index 311093aa0..b5a2d87e8 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -2,12 +2,11 @@ from uuid import uuid4, UUID import os -from hdmf.backends.hdf5 import H5DataIO from hdmf.container import AbstractContainer, Container, Data, HERDManager from hdmf.common.resources import HERD from hdmf.testing import TestCase from hdmf.utils import docval -from hdmf.common import (DynamicTable, VectorData, DynamicTableRegion) +from hdmf.common import DynamicTable, VectorData, DynamicTableRegion from hdmf.backends.hdf5.h5tools import HDF5IO @@ -397,29 +396,6 @@ def test_get_ancestors(self): self.assertTupleEqual(parent_obj.get_ancestors(), (grandparent_obj, )) self.assertTupleEqual(child_obj.get_ancestors(), (parent_obj, grandparent_obj)) - def test_set_data_io(self): - - class ContainerWithData(Container): - __fields__ = ('data1', 'data2') - - @docval( - {"name": "name", "doc": "name", "type": str}, - {'name': 'data1', 'doc': 'field1 doc', 'type': list}, - {'name': 'data2', 'doc': 'field2 doc', 'type': list, 'default': None} - ) - def __init__(self, **kwargs): - super().__init__(name=kwargs["name"]) - self.data1 = kwargs["data1"] - self.data2 = kwargs["data2"] - - obj = ContainerWithData("name", [1, 2, 3, 4, 5], None) - obj.set_data_io("data1", H5DataIO, chunks=True) - assert isinstance(obj.data1, H5DataIO) - - with self.assertRaises(ValueError): - obj.set_data_io("data2", H5DataIO, chunks=True) - - class TestHTMLRepr(TestCase): @@ -446,29 +422,32 @@ def test_repr_html_(self): child_obj1 = Container('test child 1') obj1 = self.ContainerWithChildAndData(child=child_obj1, data=[1, 2, 3], str="hello") assert obj1._repr_html_() == ( - '\n \n \n \n' - '

test ' - 'name (ContainerWithChildAndData)

child
data
1
2
3
<' - 'div style="margin-left: 0px;" class="container-fields">st' - 'r: hello
' + '\n \n \n ' + '\n

test name (' + 'ContainerWithChildAndData)

child
data
0: 1
1: 2
2: ' + '3
str: hello
' ) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 90934df94..5a4fd5a32 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -20,7 +20,7 @@ from hdmf.backends.errors import UnsupportedOperation from hdmf.build import GroupBuilder, DatasetBuilder, BuildManager, TypeMap, OrphanContainerBuildError, LinkBuilder from hdmf.container import Container -from hdmf import Data +from hdmf import Data, docval from hdmf.data_utils import DataChunkIterator, GenericDataChunkIterator, InvalidDataIOError from hdmf.spec.catalog import SpecCatalog from hdmf.spec.namespace import NamespaceCatalog, SpecNamespace @@ -3671,3 +3671,57 @@ def test_hdf5io_can_read(): assert not HDF5IO.can_read("not_a_file") assert HDF5IO.can_read("tests/unit/back_compat_tests/1.0.5.h5") assert not HDF5IO.can_read(__file__) # this file is not an HDF5 file + + +class TestContainerSetDataIO(TestCase): + + def setUp(self) -> None: + class ContainerWithData(Container): + __fields__ = ('data1', 'data2') + + @docval( + {"name": "name", "doc": "name", "type": str}, + {'name': 'data1', 'doc': 'field1 doc', 'type': list}, + {'name': 'data2', 'doc': 'field2 doc', 'type': list, 'default': None} + ) + def __init__(self, **kwargs): + super().__init__(name=kwargs["name"]) + self.data1 = kwargs["data1"] + self.data2 = kwargs["data2"] + + self.obj = ContainerWithData("name", [1, 2, 3, 4, 5], None) + + def test_set_data_io(self): + self.obj.set_data_io("data1", H5DataIO, data_io_kwargs=dict(chunks=True)) + assert isinstance(self.obj.data1, H5DataIO) + assert self.obj.data1.io_settings["chunks"] + + def test_fail_set_data_io(self): + """Attempt to set a DataIO for a dataset that is missing.""" + with self.assertRaisesWith(ValueError, "data2 is None and cannot be wrapped in a DataIO class"): + self.obj.set_data_io("data2", H5DataIO, data_io_kwargs=dict(chunks=True)) + + def test_set_data_io_old_api(self): + """Test that using the kwargs still works but throws a warning.""" + msg = ( + "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a dictionary to" + " the `data_io_kwargs` parameter instead." + ) + with self.assertWarnsWith(DeprecationWarning, msg): + self.obj.set_data_io("data1", H5DataIO, chunks=True) + self.assertIsInstance(self.obj.data1, H5DataIO) + self.assertTrue(self.obj.data1.io_settings["chunks"]) + + +class TestDataSetDataIO(TestCase): + + def setUp(self): + class MyData(Data): + pass + + self.data = MyData("my_data", [1, 2, 3]) + + def test_set_data_io(self): + self.data.set_data_io(H5DataIO, dict(chunks=True)) + assert isinstance(self.data.data, H5DataIO) + assert self.data.data.io_settings["chunks"] diff --git a/tests/unit/test_multicontainerinterface.py b/tests/unit/test_multicontainerinterface.py index 4b1dc0c87..c705d0a6e 100644 --- a/tests/unit/test_multicontainerinterface.py +++ b/tests/unit/test_multicontainerinterface.py @@ -330,26 +330,29 @@ def test_repr_html_(self): self.assertEqual( foo._repr_html_(), ( - '\n \n \n \n

FooSingle

containers (2)
obj1
obj2
' + '\n \n \n ' + ' \n

FooSingle

containers
obj1
obj2
' ) ) diff --git a/tests/unit/utils_test/test_core_DataIO.py b/tests/unit/utils_test/test_core_DataIO.py index 00941cb0e..778dd2617 100644 --- a/tests/unit/utils_test/test_core_DataIO.py +++ b/tests/unit/utils_test/test_core_DataIO.py @@ -8,12 +8,6 @@ class DataIOTests(TestCase): - def setUp(self): - pass - - def tearDown(self): - pass - def test_copy(self): obj = DataIO(data=[1., 2., 3.]) obj_copy = copy(obj) diff --git a/tests/unit/validator_tests/test_validate.py b/tests/unit/validator_tests/test_validate.py index 4af4dfd16..7002ebd6f 100644 --- a/tests/unit/validator_tests/test_validate.py +++ b/tests/unit/validator_tests/test_validate.py @@ -116,7 +116,18 @@ def getSpecs(self): ), DatasetSpec('an example time dataset', 'isodatetime', name='datetime'), DatasetSpec('an example time dataset', 'isodatetime', name='date', quantity='?'), - DatasetSpec('an array of times', 'isodatetime', name='time_array', dims=('num_times',), shape=(None,)) + DatasetSpec('an array of times', 'isodatetime', name='time_array', dims=('num_times',), shape=(None,)), + DatasetSpec( + doc='an array with compound dtype that includes an isodatetime', + dtype=[ + DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='isodatetime'), + ], + name='cpd_array', + dims=('num_times',), + shape=(None,), + quantity="?", + ), ], attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')]) return ret, @@ -129,7 +140,15 @@ def test_valid_isodatetime(self): DatasetBuilder('data', 100, attributes={'attr2': 10}), DatasetBuilder('datetime', datetime(2017, 5, 1, 12, 0, 0)), DatasetBuilder('date', date(2017, 5, 1)), - DatasetBuilder('time_array', [datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())]) + DatasetBuilder('time_array', [datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())]), + DatasetBuilder( + name='cpd_array', + data=[(1, datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal()))], + dtype=[ + DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='isodatetime'), + ], + ), ] ) validator = self.vmap.get_validator('Bar') @@ -143,7 +162,7 @@ def test_invalid_isodatetime(self): datasets=[ DatasetBuilder('data', 100, attributes={'attr2': 10}), DatasetBuilder('datetime', 100), - DatasetBuilder('time_array', [datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())]) + DatasetBuilder('time_array', [datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())]), ] ) validator = self.vmap.get_validator('Bar') @@ -152,18 +171,44 @@ def test_invalid_isodatetime(self): self.assertValidationError(result[0], DtypeError, name='Bar/datetime') def test_invalid_isodatetime_array(self): - builder = GroupBuilder('my_bar', - attributes={'data_type': 'Bar', 'attr1': 'a string attribute'}, - datasets=[DatasetBuilder('data', 100, attributes={'attr2': 10}), - DatasetBuilder('datetime', - datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())), - DatasetBuilder('time_array', - datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal()))]) + builder = GroupBuilder( + 'my_bar', + attributes={'data_type': 'Bar', 'attr1': 'a string attribute'}, + datasets=[ + DatasetBuilder('data', 100, attributes={'attr2': 10}), + DatasetBuilder('datetime', datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())), + DatasetBuilder('time_array', datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())), + ], + ) validator = self.vmap.get_validator('Bar') result = validator.validate(builder) self.assertEqual(len(result), 1) self.assertValidationError(result[0], ExpectedArrayError, name='Bar/time_array') + def test_invalid_cpd_isodatetime_array(self): + builder = GroupBuilder( + 'my_bar', + attributes={'data_type': 'Bar', 'attr1': 'a string attribute'}, + datasets=[ + DatasetBuilder('data', 100, attributes={'attr2': 10}), + DatasetBuilder('datetime', datetime(2017, 5, 1, 12, 0, 0)), + DatasetBuilder('date', date(2017, 5, 1)), + DatasetBuilder('time_array', [datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())]), + DatasetBuilder( + name='cpd_array', + data=[(1, "wrong")], + dtype=[ + DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='isodatetime'), + ], + ), + ], + ) + validator = self.vmap.get_validator('Bar') + result = validator.validate(builder) + self.assertEqual(len(result), 1) + self.assertValidationError(result[0], DtypeError, name='Bar/cpd_array') + class TestNestedTypes(ValidatorTestBase): @@ -508,6 +553,58 @@ def test_empty_nparray(self): # TODO test shape validation more completely +class TestStringDatetime(TestCase): + + def test_str_coincidental_isodatetime(self): + """Test validation of a text spec allows a string that coincidentally matches the isodatetime format.""" + spec_catalog = SpecCatalog() + spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec(doc='an example scalar dataset', dtype="text", name='data1'), + DatasetSpec(doc='an example 1D dataset', dtype="text", name='data2', shape=(None, )), + DatasetSpec( + doc='an example 1D compound dtype dataset', + dtype=[ + DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='text'), + ], + name='data3', + shape=(None, ), + ), + ], + attributes=[ + AttributeSpec(name='attr1', doc='an example scalar attribute', dtype="text"), + AttributeSpec(name='attr2', doc='an example 1D attribute', dtype="text", shape=(None, )), + ] + ) + spec_catalog.register_spec(spec, 'test.yaml') + namespace = SpecNamespace( + 'a test namespace', CORE_NAMESPACE, [{'source': 'test.yaml'}], version='0.1.0', catalog=spec_catalog + ) + vmap = ValidatorMap(namespace) + + bar_builder = GroupBuilder( + name='my_bar', + attributes={'data_type': 'Bar', 'attr1': "2023-01-01", 'attr2': ["2023-01-01"]}, + datasets=[ + DatasetBuilder(name='data1', data="2023-01-01"), + DatasetBuilder(name='data2', data=["2023-01-01"]), + DatasetBuilder( + name='data3', + data=[(1, "2023-01-01")], + dtype=[ + DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='text'), + ], + ), + ], + ) + results = vmap.validate(bar_builder) + self.assertEqual(len(results), 0) + + class TestLinkable(TestCase): def set_up_spec(self): diff --git a/tox.ini b/tox.ini index 596262002..aeb743c45 100644 --- a/tox.ini +++ b/tox.ini @@ -1,193 +1,59 @@ # Tox (https://tox.readthedocs.io/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. +# and then run "tox -e [envname]" from this directory. [tox] -envlist = py38, py39, py310, py311 requires = pip >= 22.0 [testenv] download = True usedevelop = True setenv = - PYTHONDONTWRITEBYTECODE = 1 - VIRTUALENV_PIP = 22.3.1 -install_command = - python -m pip install {opts} {packages} - -deps = - -rrequirements-dev.txt - -rrequirements.txt -commands = - python -m pip check # Check for conflicting packages - python -m pip list - pytest -v - -# Test with python 3.11; pinned dev and optional reqs -[testenv:py311-optional] -basepython = python3.11 -deps = - {[testenv]deps} - -rrequirements-opt.txt -commands = {[testenv]commands} - -# Test with python 3.11; pinned dev and optional reqs; upgraded run reqs -[testenv:py311-upgraded] -basepython = python3.11 -install_command = - python -m pip install -U {opts} {packages} -deps = - -rrequirements-dev.txt - -rrequirements-opt.txt -commands = {[testenv]commands} - -# Test with python 3.11; pinned dev and optional reqs; upgraded, pre-release run reqs -[testenv:py311-prerelease] -basepython = python3.11 -install_command = - python -m pip install -U --pre {opts} {packages} -deps = - -rrequirements-dev.txt - -rrequirements-opt.txt -commands = {[testenv]commands} - -# Test with python 3.8; pinned dev reqs; minimum run reqs -[testenv:py38-minimum] -basepython = python3.8 -deps = - -rrequirements-dev.txt - -rrequirements-min.txt -commands = {[testenv]commands} - -# Envs that builds wheels and source distribution -[testenv:build] + PYTHONDONTWRITEBYTECODE = 1 + VIRTUALENV_PIP = 23.3.1 +recreate = + pinned, minimum, upgraded, prerelease: False + build, wheelinstall: True # good practice to recreate the environment +skip_install = + pinned, minimum, upgraded, prerelease, wheelinstall: False + build: True # no need to install anything when building +install_command = + # when using [testenv:wheelinstall] and --installpkg, the wheel and its dependencies + # are installed, instead of the package in the current directory + pinned, minimum, wheelinstall: python -I -m pip install {opts} {packages} + upgraded: python -I -m pip install -U {opts} {packages} + prerelease: python -I -m pip install -U --pre {opts} {packages} +deps = + # use pinned, minimum, or neither (use dependencies in pyproject.toml) + pytest, gallery: -rrequirements-dev.txt + gallery: -rrequirements-doc.txt + optional: -rrequirements-opt.txt + pinned: -rrequirements.txt + minimum: -rrequirements-min.txt commands = - python -m pip install --upgrade build - python -m build - -[testenv:build-py38] -basepython = python3.8 -commands = {[testenv:build]commands} - -[testenv:build-py39] -basepython = python3.9 -commands = {[testenv:build]commands} - -[testenv:build-py310] -basepython = python3.10 -commands = {[testenv:build]commands} - -[testenv:build-py311] -basepython = python3.11 -commands = {[testenv:build]commands} - -[testenv:build-py311-optional] -basepython = python3.11 -deps = - {[testenv]deps} - -rrequirements-opt.txt -commands = {[testenv:build]commands} - -[testenv:build-py311-upgraded] -basepython = python3.11 -install_command = - python -m pip install -U {opts} {packages} -deps = - -rrequirements-dev.txt - -rrequirements-opt.txt -commands = {[testenv:build]commands} - -[testenv:build-py311-prerelease] -basepython = python3.11 -install_command = - python -m pip install -U --pre {opts} {packages} -deps = - -rrequirements-dev.txt - -rrequirements-opt.txt -commands = {[testenv:build]commands} - -[testenv:build-py38-minimum] -basepython = python3.8 -deps = - -rrequirements-dev.txt - -rrequirements-min.txt -commands = {[testenv:build]commands} - -# Envs that will test installation from a wheel -[testenv:wheelinstall] -deps = null -commands = python -c "import hdmf; import hdmf.common" - -# Envs that will execute gallery tests -[testenv:gallery] -install_command = - python -m pip install {opts} {packages} - -deps = - -rrequirements-dev.txt - -rrequirements.txt - -rrequirements-doc.txt - -commands = - python test_gallery.py - -[testenv:gallery-py38] -basepython = python3.8 -deps = {[testenv:gallery]deps} -commands = {[testenv:gallery]commands} - -[testenv:gallery-py39] -basepython = python3.9 -deps = {[testenv:gallery]deps} -commands = {[testenv:gallery]commands} - -[testenv:gallery-py310] -basepython = python3.10 -deps = {[testenv:gallery]deps} -commands = {[testenv:gallery]commands} - -[testenv:gallery-py311] -basepython = python3.11 -deps = {[testenv:gallery]deps} -commands = {[testenv:gallery]commands} - -[testenv:gallery-py311-optional] -basepython = python3.11 -deps = - -rrequirements-dev.txt - -rrequirements.txt - -rrequirements-doc.txt - -rrequirements-opt.txt -commands = {[testenv:gallery]commands} - -# Test with python 3.11; pinned dev, doc, and optional reqs; upgraded run reqs -[testenv:gallery-py311-upgraded] -basepython = python3.11 -install_command = - python -m pip install -U {opts} {packages} -deps = - -rrequirements-dev.txt - -rrequirements-doc.txt - -rrequirements-opt.txt -commands = {[testenv:gallery]commands} - -# Test with python 3.11; pinned dev, doc, and optional reqs; pre-release run reqs -[testenv:gallery-py311-prerelease] -basepython = python3.11 -install_command = - python -m pip install -U --pre {opts} {packages} -deps = - -rrequirements-dev.txt - -rrequirements-doc.txt - -rrequirements-opt.txt -commands = {[testenv:gallery]commands} - -# Test with python 3.8; pinned dev and doc reqs; minimum run reqs + python --version # print python version for debugging + python -m pip check # check for conflicting packages + python -m pip list # list installed packages for debugging + pytest: pytest -v + gallery: python test_gallery.py + build: python -m pip install -U build + build: python -m build + wheelinstall: python -c "import hdmf; import hdmf.common" + +# list of pre-defined environments. (Technically environments not listed here +# like build-py312 can also be used.) +[testenv:pytest-py312-upgraded] +[testenv:pytest-py312-prerelease] +[testenv:pytest-py311-optional-pinned] # some optional reqs not compatible with py312 yet +[testenv:pytest-py{38,39,310,311,312}-pinned] +[testenv:pytest-py38-minimum] + +[testenv:gallery-py312-upgraded] +[testenv:gallery-py312-prerelease] +[testenv:gallery-py311-optional-pinned] +[testenv:gallery-py{38,39,310,311,312}-pinned] [testenv:gallery-py38-minimum] -basepython = python3.8 -deps = - -rrequirements-dev.txt - -rrequirements-min.txt - -rrequirements-doc.txt -commands = {[testenv:gallery]commands} + +[testenv:build] # using tox for this so that we can have a clean build environment +[testenv:wheelinstall] # use with `--installpkg dist/*-none-any.whl`