diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e12e5df..3415edd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,103 +2,137 @@ name: packages on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+' + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+" # Dry-run only workflow_dispatch: - inputs: - target: - description: 'Build mode' - type: choice - options: - - dryrun - required: true - default: dryrun schedule: - - cron: '0 01 * * SUN' + - cron: "0 16 * * SUN" + +defaults: + run: + shell: bash -el {0} + +env: + PYTHON_VERSION: "3.11" + PACKAGE: "spatialpandas" jobs: + waiting_room: + name: Waiting Room + runs-on: ubuntu-latest + needs: [conda_build, pip_install] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + environment: + name: publish + steps: + - run: echo "All builds have finished, have been approved, and ready to publish" + + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@pixi + conda_build: - name: Build Conda Packages - runs-on: 'ubuntu-latest' - defaults: - run: - shell: bash -l {0} - env: - PKG_TEST_PYTHON: "--test-python=py39" - CONDA_UPLOAD_TOKEN: ${{ secrets.CONDA_UPLOAD_TOKEN }} + name: Build Conda + needs: [pixi_lock] + runs-on: "ubuntu-latest" + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_install@pixi + with: + environments: "build" + download-data: false + install: false + - name: conda build + run: pixi run -e build build-conda + - uses: actions/upload-artifact@v4 + if: always() + with: + name: conda + path: dist/*.tar.bz2 + if-no-files-found: error + + conda_publish: + name: Publish Conda + runs-on: ubuntu-latest + needs: [conda_build, waiting_room] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/checkout@v3 + - uses: actions/download-artifact@v4 with: - fetch-depth: "100" - - uses: conda-incubator/setup-miniconda@v2 + name: conda + path: dist/ + - name: Set environment variables + run: | + echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV + - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" - python-version: 3.9 - channels: pyviz/label/dev,conda-forge,nodefaults - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - name: Set output - id: vars - run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - name: conda setup run: | - conda install -c pyviz "pyctdev>=0.5" - doit ecosystem_setup - # FIXME: downgrade urllib3 until this issue is fixed: - # https://github.com/Anaconda-Platform/anaconda-client/issues/654 - conda install -c conda-forge "urllib3<2.0.0" - - name: doit env_capture - run: | - doit env_capture - - name: conda build - run: doit package_build $PKG_TEST_PYTHON --no-pkg-tests + conda install -y anaconda-client - name: conda dev upload - if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) - run: doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev + if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') + run: | + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE - name: conda main upload - if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) - run: doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev --label=main + if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) + run: | + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + pip_build: - name: Build PyPI Packages - runs-on: 'ubuntu-latest' - defaults: - run: - shell: bash -l {0} - env: - CHANS_DEV: "-c pyviz/label/dev -c conda-forge" - PKG_TEST_PYTHON: "--test-python=py39" - PYTHON_VERSION: "3.9" - CHANS: "-c pyviz" - PPU: ${{ secrets.PPU }} - PPP: ${{ secrets.PPP }} - PYPI: "https://upload.pypi.org/legacy/" + name: Build PyPI + needs: [pixi_lock] + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v3 - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - uses: conda-incubator/setup-miniconda@v2 + - uses: holoviz-dev/holoviz_tasks/pixi_install@pixi with: - miniconda-version: "latest" - - name: conda setup - run: | - conda config --set always_yes True - conda install -c pyviz "pyctdev>=0.5" - doit ecosystem_setup - doit env_create $CHANS_DEV --python=$PYTHON_VERSION - - name: env setup - run: | - conda activate test-environment - doit develop_install $CHANS_DEV -o tests - doit pip_on_conda - - name: pip build - run: | - conda activate test-environment - doit ecosystem=pip package_build $PKG_TEST_PYTHON --no-pkg-test - - name: pip upload - if: github.event_name == 'push' - run: | - conda activate test-environment - doit ecosystem=pip package_upload -u $PPU -p $PPP -r $PYPI + environments: "build" + download-data: false + install: false + - name: Build package + run: pixi run -e build build-pip + - uses: actions/upload-artifact@v4 + if: always() + with: + name: pip + path: dist/ + if-no-files-found: error + + pip_install: + name: Install PyPI + runs-on: "ubuntu-latest" + needs: [pip_build] + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + - name: Install package + run: python -m pip install dist/*.whl + - name: Import package + run: python -c "import $PACKAGE; print($PACKAGE.__version__)" + + pip_publish: + name: Publish PyPI + runs-on: ubuntu-latest + needs: [pip_build, waiting_room] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: ${{ secrets.PPU }} + password: ${{ secrets.PPP }} + repository-url: "https://upload.pypi.org/legacy/" diff --git a/.github/workflows/nightly_lock.yaml b/.github/workflows/nightly_lock.yaml new file mode 100644 index 0000000..8d9921d --- /dev/null +++ b/.github/workflows/nightly_lock.yaml @@ -0,0 +1,25 @@ +name: nightly_lock +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +env: + PACKAGE: "spatialpandas" + +jobs: + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@pixi + - name: Upload lock-file to S3 + if: "!github.event.pull_request.head.repo.fork" + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" + run: | + zip $(date +%Y-%m-%d).zip pixi.lock pixi.toml + aws s3 cp ./$(date +%Y-%m-%d).zip s3://assets.holoviz.org/lock/$PACKAGE/ diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index db37f5f..3a2bc7e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,50 +5,154 @@ on: - main pull_request: branches: - - '*' + - "*" workflow_dispatch: + inputs: + target: + description: "How much of the test suite to run" + type: choice + default: default + options: + - default + - full + - downstream + cache: + description: "Use cache" + type: boolean + default: true + schedule: - - cron: '0 01 * * SUN' + - cron: "0 01 * * SUN" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash -el {0} + env: - COLUMNS: 120 + DISPLAY: ":99.0" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COV: "--cov=./spatialpandas --cov-report=xml" jobs: - test_suite: - name: Tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} + pre_commit: + name: Run pre-commit + runs-on: "ubuntu-latest" + steps: + - uses: holoviz-dev/holoviz_tasks/pre-commit@v0 + + setup: + name: Setup workflow + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + code_change: ${{ steps.filter.outputs.code }} + matrix: ${{ env.MATRIX }} + steps: + - uses: actions/checkout@v4 + if: github.event_name != 'pull_request' + - name: Check for code changes + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + code: + - 'spatialpandas/**' + - 'examples/**' + - 'pixi.toml' + - 'pyproject.toml' + - '.github/workflows/test.yaml' + - name: Set matrix option + run: | + if [[ '${{ github.event_name }}' == 'workflow_dispatch' ]]; then + OPTION=${{ github.event.inputs.target }} + elif [[ '${{ github.event_name }}' == 'schedule' ]]; then + OPTION="full" + elif [[ '${{ github.event_name }}' == 'push' && '${{ github.ref_type }}' == 'tag' ]]; then + OPTION="full" + else + OPTION="default" + fi + echo "MATRIX_OPTION=$OPTION" >> $GITHUB_ENV + - name: Set test matrix with 'default' option + if: env.MATRIX_OPTION == 'default' + run: | + MATRIX=$(jq -nsc '{ + "os": ["ubuntu-latest", "macos-latest", "windows-latest"], + "environment": ["test-39", "test-312"] + }') + echo "MATRIX=$MATRIX" >> $GITHUB_ENV + - name: Set test matrix with 'full' option + if: env.MATRIX_OPTION == 'full' + run: | + MATRIX=$(jq -nsc '{ + "os": ["ubuntu-latest", "macos-latest", "windows-latest"], + "environment": ["test-39", "test-310", "test-311", "test-312"] + }') + echo "MATRIX=$MATRIX" >> $GITHUB_ENV + - name: Set test matrix with 'downstream' option + if: env.MATRIX_OPTION == 'downstream' + run: | + MATRIX=$(jq -nsc '{ + "os": ["ubuntu-latest"], + "environment": ["test-311"] + }') + echo "MATRIX=$MATRIX" >> $GITHUB_ENV + + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@pixi + with: + cache: ${{ github.event.inputs.cache == 'true' || github.event.inputs.cache == '' }} + + unit_test_suite: + name: unit:${{ matrix.environment }}:${{ matrix.os }} + needs: [pre_commit, setup, pixi_lock] runs-on: ${{ matrix.os }} strategy: fail-fast: false - matrix: - os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ["3.9", "3.10", "3.11"] - timeout-minutes: 90 - defaults: - run: - shell: bash -l {0} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + timeout-minutes: 120 steps: - - uses: holoviz-dev/holoviz_tasks/install@v0.1a15 + - uses: holoviz-dev/holoviz_tasks/pixi_install@pixi + if: needs.setup.outputs.code_change == 'true' with: - name: unit_test_suite - python-version: ${{ matrix.python-version }} - channel-priority: strict - channels: pyviz/label/dev,conda-forge,nodefaults - envs: "-o tests" - cache: true - conda-update: true - id: install - - name: doit test_lint - if: runner.os != 'Windows' + environments: ${{ matrix.environment }} + - name: Test Unit + if: needs.setup.outputs.code_change == 'true' run: | - conda activate test-environment - doit test_lint - - name: doit test_unit_deploy + pixi run -e ${{ matrix.environment }} test-unit $COV + - name: Test Examples + if: needs.setup.outputs.code_change == 'true' + run: | + pixi run -e ${{ matrix.environment }} test-example + - uses: codecov/codecov-action@v4 + if: needs.setup.outputs.code_change == 'true' + with: + token: ${{ secrets.CODECOV_TOKEN }} + + core_test_suite: + name: core:${{ matrix.environment }}:${{ matrix.os }} + needs: [pre_commit, setup, pixi_lock] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + environment: ["test-core"] + timeout-minutes: 120 + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_install@pixi + if: needs.setup.outputs.code_change == 'true' + with: + environments: ${{ matrix.environment }} + - name: Test Unit + if: needs.setup.outputs.code_change == 'true' run: | - conda activate test-environment - doit test_unit_deploy + pixi run -e ${{ matrix.environment }} test-unit diff --git a/.gitignore b/.gitignore index a4a2b42..1048316 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,12 @@ spatialpandas/.version monkeytype.sqlite3 .doit.db + +# pixi + hatch +.pixi +pixi.lock +_version.py + +# Exampel files +examples/world.parq +examples/world_packed.parq/ diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index d985c00..0000000 --- a/.isort.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[settings] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -known_local_folder=spatialpandas -known_first_party=spatialpandas diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f11f403 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +exclude: (\.(js|svg)$) + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-builtin-literals + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-toml + - id: detect-private-key + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.6 + hooks: + - id: ruff + files: spatialpandas/ + # - repo: https://github.com/codespell-project/codespell + # rev: v2.3.0 + # hooks: + # - id: codespell + # additional_dependencies: + # - tomli + # - repo: https://github.com/pycqa/isort + # rev: 5.13.2 + # hooks: + # - id: isort + # name: isort (python) + +ci: + autofix_prs: false diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index c9861d1..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -include LICENSE -include NOTICE -include README.md -include CHANGELOG.md -include spatialpandas/.version -graft spatialpandas/tests/test_data -global-exclude *.py[co] -global-exclude *~ -global-exclude *.ipynb_checkpoints/* diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml deleted file mode 100644 index c5d1eee..0000000 --- a/conda.recipe/meta.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{% set sdata = load_setup_py_data() %} - -package: - name: spatialpandas - version: {{ sdata['version'] }} - -source: - path: .. - -build: - noarch: python - script: python setup.py install --single-version-externally-managed --record=record.txt - entry_points: - {% for group,epoints in sdata.get("entry_points",{}).items() %} - {% for entry_point in epoints %} - - {{ entry_point }} - {% endfor %} - {% endfor %} - -requirements: - host: - - python {{ sdata['python_requires'] }} - - param >=1.7.0 - - setuptools >30.3.0 - run: - - python {{ sdata['python_requires'] }} - {% for dep in sdata.get('install_requires',{}) %} - - {{ dep }} - {% endfor %} -test: - imports: - - spatialpandas - - spatialpandas.dask - -about: - home: {{ sdata['url'] }} - summary: {{ sdata['description'] }} - license: {{ sdata['license'] }} diff --git a/dodo.py b/dodo.py deleted file mode 100644 index 8111b49..0000000 --- a/dodo.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -if "PYCTDEV_ECOSYSTEM" not in os.environ: - os.environ["PYCTDEV_ECOSYSTEM"] = "conda" - -from pyctdev import * # noqa: api - -def task_pip_on_conda(): - """Experimental: provide pip build env via conda""" - return {'actions':[ - # some ecosystem=pip build tools must be installed with conda when using conda... - 'conda install -y pip twine wheel', - # ..and some are only available via conda-forge - 'conda install -y -c conda-forge tox virtualenv', - ]} diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000..5932f4e --- /dev/null +++ b/pixi.toml @@ -0,0 +1,105 @@ +[project] +name = "spatialpandas" +channels = ["pyviz/label/dev", "conda-forge"] +platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] + +[tasks] +install = 'python -m pip install --no-deps --disable-pip-version-check -e .' + +[activation.env] +PYTHONIOENCODING = "utf-8" +DASK_DATAFRAME__QUERY_PLANNING = "False" # TODO: Support query planning +USE_PYGEOS = '0' + +[environments] +test-39 = ["py39", "test-core", "test", "test-task", "example", "test-example"] +test-310 = ["py310", "test-core", "test", "test-task", "example", "test-example"] +test-311 = ["py311", "test-core", "test", "test-task", "example", "test-example"] +test-312 = ["py312", "test-core", "test", "test-task", "example", "test-example"] +test-core = ["py312", "test-core", "test-core-task"] +build = ["py311", "build"] +lint = ["py311", "lint"] + +[dependencies] +dask-core = "*" +fsspec = "*" +numba = "*" +pandas = "<2.2" # FIX: Temporary upper pin +pip = "*" +pyarrow = ">=10,<15" # FIX: Temporary upper pin +retrying = "*" + +[feature.py39.dependencies] +python = "3.9.*" + +[feature.py310.dependencies] +python = "3.10.*" + +[feature.py311.dependencies] +python = "3.11.*" + +[feature.py312.dependencies] +python = "3.12.*" + +[feature.example.dependencies] +datashader = "*" +descartes = "*" +distributed = "*" +geopandas-base = "<1" # FIX: temporary upper pin +pyogrio = "*" +holoviews = "*" +matplotlib-base = "*" + +# ============================================= +# =================== TESTS =================== +# ============================================= +[feature.test-core.dependencies] +hypothesis = "*" +psutil = "*" +pytest = "*" +pytest-cov = "*" +pytest-github-actions-annotate-failures = "*" +pytest-xdist = "*" + +[feature.test-core-task.tasks] +test-unit = 'pytest spatialpandas/tests -n logical --dist loadgroup --skip-slow' + +[feature.test.dependencies] +geopandas-base = "*" +hilbertcurve = "*" +moto = "*" +python-snappy = "*" +rtree = "*" +s3fs = ">=2022.8" +scipy = "*" +shapely = "*" + +[feature.test-task.tasks] +test-unit = 'pytest spatialpandas/tests -n logical --dist loadgroup' + +[feature.test-example.dependencies] +nbval = "*" + +[feature.test-example.tasks] +test-example = 'pytest -n logical --dist loadscope --nbval-lax examples' + +# ============================================= +# ================== BUILD ==================== +# ============================================= +[feature.build.dependencies] +python-build = "*" +conda-build = "*" + +[feature.build.tasks] +build-conda = 'bash scripts/conda/build.sh' +build-pip = 'python -m build .' + +# ============================================= +# =================== LINT ==================== +# ============================================= +[feature.lint.dependencies] +pre-commit = "*" + +[feature.lint.tasks] +lint = 'pre-commit run --all-files' +lint-install = 'pre-commit install' diff --git a/pyproject.toml b/pyproject.toml index fec98ab..dd6b5cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,97 @@ [build-system] -requires = [ - "param >=1.7.0", - "pyct >=0.4.4", - "setuptools >=30.3.0" +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "spatialpandas" +dynamic = ["version"] +description = 'Pandas extension arrays for spatial/geometric operations' +readme = "README.md" +license = { text = "BSD-2-Clause" } +requires-python = ">=3.9" +authors = [{ name = "HoloViz developers", email = "developers@holoviz.org" }] +maintainers = [{ name = "HoloViz developers", email = "developers@holoviz.org" }] +classifiers = [ + "License :: OSI Approved :: BSD License", + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Natural Language :: English", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries", ] +dependencies = ['dask', 'fsspec >=2022.8', 'numba', 'pandas', 'pyarrow >=10', 'retrying'] + +[project.urls] +Homepage = "https://github.com/holoviz/spatialpandas" +Source = "https://github.com/holoviz/spatialpandas" +HoloViz = "https://holoviz.org/" + +[project.optional-dependencies] +tests = ['pytest', 'hypothesis'] + +[tool.hatch.version] +source = "vcs" +raw-options = { version_scheme = "no-guess-dev" } + +[tool.hatch.build.targets.wheel] +include = ["spatialpandas"] + +[tool.hatch.build.targets.sdist] +include = ["spatialpandas", "scripts", "examples"] + +[tool.hatch.build.hooks.vcs] +version-file = "spatialpandas/_version.py" + +[tool.pytest.ini_options] +addopts = [ + "--pyargs", + "--doctest-modules", + "--doctest-ignore-import-errors", + "--strict-config", + "--strict-markers", + "--color=yes", +] +minversion = "7" +xfail_strict = true +log_cli_level = "INFO" +filterwarnings = [] + +[tool.ruff] +fix = true +line-length = 100 + +[tool.ruff.lint] +ignore = [ + "E402", # Module level import not at top of file + "E501", # Line too long + "E701", # Multiple statements on one line + "E712", # Comparison to true should be is + "E731", # Do not assign a lambda expression, use a def + "E741", # Ambiguous variable name + "F405", # From star imports + # "PLE0604", # Invalid object in `__all__`, must contain only strings + # "PLE0605", # Invalid format for `__all__` + # "PLR091", # Too many arguments/branches/statements + # "PLR2004", # Magic value used in comparison + # "PLW2901", # `for` loop variable is overwritten + # "RUF005", # Consider {expr} instead of concatenation + # "RUF012", # Mutable class attributes should use `typing.ClassVar` +] +extend-unsafe-fixes = [ + "F401", # Unused imports + "F841", # Unused variables +] + +[tool.isort] +force_grid_wrap = 4 +multi_line_output = 5 +combine_as_imports = true +lines_between_types = 1 +include_trailing_comma = true diff --git a/scripts/conda/build.sh b/scripts/conda/build.sh new file mode 100755 index 0000000..00f5cac --- /dev/null +++ b/scripts/conda/build.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +PACKAGE="spatialpandas" + +python -m build -w . + +VERSION=$(python -c "import $PACKAGE; print($PACKAGE._version.__version__)") +export VERSION + +# conda config --env --set conda_build.pkg_format 2 +conda build scripts/conda/recipe --no-anaconda-upload --no-verify + +# mv "$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.conda" dist +mv "$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2" dist diff --git a/scripts/conda/recipe/meta.yaml b/scripts/conda/recipe/meta.yaml new file mode 100644 index 0000000..8302186 --- /dev/null +++ b/scripts/conda/recipe/meta.yaml @@ -0,0 +1,45 @@ +{% set pyproject = load_file_data('../../../pyproject.toml', from_recipe_dir=True) %} +{% set project = pyproject['project'] %} + +package: + name: {{ project["name"] }} + version: {{ VERSION }} + +source: + url: ../../../dist/{{ project["name"] }}-{{ VERSION }}-py3-none-any.whl + +build: + noarch: python + script: {{ PYTHON }} -m pip install --no-deps -vv {{ project["name"] }}-{{ VERSION }}-py3-none-any.whl + entry_points: + {% for group,epoints in project.get("entry_points",{}).items() %} + {% for entry_point in epoints %} + - {{ entry_point }} + {% endfor %} + {% endfor %} + +requirements: + build: + - python {{ project['requires-python'] }} + {% for dep in pyproject['build-system']['requires'] %} + - {{ dep }} + {% endfor %} + run: + - python {{ project['requires-python'] }} + {% for dep in project.get('dependencies', []) %} + - {{ dep.replace('dask', 'dask-core') }} + {% endfor %} + +test: + imports: + - spatialpandas + - spatialpandas.dask + commands: + - pip check + requires: + - pip + +about: + home: {{ project['urls']['Homepage'] }} + summary: {{ project['description'] }} + license: {{ project['license']['text'] }} diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 89b9e8a..0000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[metadata] -license_files = LICENSE - -[wheel] -universal = 1 - -[tool:pyctdev.conda] -namespace_map = - dask=dask-core - geopandas=geopandas-base diff --git a/setup.py b/setup.py deleted file mode 100644 index b922f8f..0000000 --- a/setup.py +++ /dev/null @@ -1,80 +0,0 @@ -import param - -from setuptools import find_namespace_packages, setup - -extras_require = { - 'tests': [ - 'codecov', - 'flake8', - 'hilbertcurve', - 'geopandas', - 'hypothesis', - 'keyring', - 'moto[s3,server]', - 'pytest-cov', - 'pytest', - 'python-snappy', - 'rfc3986', - 's3fs', - 'scipy', - 'shapely', - 'twine', - ], - 'examples': [ - 'datashader', - 'distributed', - 'descartes', - 'geopandas', - 'holoviews', - 'matplotlib', - ] -} - -install_requires = [ - 'dask', - 'fsspec', - 'numba', - 'pandas', - 'param', - 'pyarrow >=1.0', - 'retrying', -] - -setup_args = dict( - name='spatialpandas', - version=param.version.get_setup_version( - __file__, - "spatialpandas", - archive_commit="$Format:%h$", - ), - description='Pandas extension arrays for spatial/geometric operations', - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - url='https://github.com/holoviz/spatialpandas', - maintainer='HoloViz developers', - maintainer_email='developers@holoviz.org', - python_requires='>=3.9', - install_requires=install_requires, - extras_require=extras_require, - tests_require=extras_require['tests'], - license='BSD-2-Clause', - packages=find_namespace_packages(), - include_package_data=True, - classifiers=[ - "License :: OSI Approved :: BSD License", - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Operating System :: OS Independent", - "Intended Audience :: Science/Research", - "Intended Audience :: Developers", - "Natural Language :: English", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries", - ], -) - -if __name__ == '__main__': - setup(**setup_args) diff --git a/spatialpandas/__init__.py b/spatialpandas/__init__.py index 138c031..0eb07a8 100644 --- a/spatialpandas/__init__.py +++ b/spatialpandas/__init__.py @@ -1,6 +1,5 @@ -import param as _param - from . import geometry, spatialindex, tools +from .__version import __version__ from .geodataframe import GeoDataFrame from .geoseries import GeoSeries from .tools.sjoin import sjoin @@ -14,16 +13,11 @@ # Dask dataframe not available pass -__version__ = str( - _param.version.Version( - fpath=__file__, - archive_commit="$Format:%h$", - reponame="spatialpandas", - )) __all__ = [ "GeoDataFrame", "GeoSeries", + "__version__", "geometry", "sjoin", "spatialindex", diff --git a/spatialpandas/__version.py b/spatialpandas/__version.py new file mode 100644 index 0000000..2987ce6 --- /dev/null +++ b/spatialpandas/__version.py @@ -0,0 +1,44 @@ +"""Define the package version. + +Called __version.py as setuptools_scm will create a _version.py +""" + +import os.path + +PACKAGE = "spatialpandas" + +try: + # For performance reasons on imports, avoid importing setuptools_scm + # if not in a .git folder + if os.path.exists(os.path.join(os.path.dirname(__file__), "..", ".git")): + # If setuptools_scm is installed (e.g. in a development environment with + # an editable install), then use it to determine the version dynamically. + from setuptools_scm import get_version + + # This will fail with LookupError if the package is not installed in + # editable mode or if Git is not installed. + __version__ = get_version(root="..", relative_to=__file__) + else: + raise FileNotFoundError +except (ImportError, LookupError, FileNotFoundError): + # As a fallback, use the version that is hard-coded in the file. + try: + # __version__ was added in _version in setuptools-scm 7.0.0, we rely on + # the hopefully stable version variable. + from ._version import version as __version__ + except (ModuleNotFoundError, ImportError): + # Either _version doesn't exist (ModuleNotFoundError) or version isn't + # in _version (ImportError). ModuleNotFoundError is a subclass of + # ImportError, let's be explicit anyway. + + # Try something else: + from importlib.metadata import PackageNotFoundError, version + + try: + __version__ = version(PACKAGE) + except PackageNotFoundError: + # The user is probably trying to run this without having installed + # the package. + __version__ = "0.0.0+unknown" + +__all__ = ("__version__",) diff --git a/spatialpandas/geometry/base.py b/spatialpandas/geometry/base.py index 6f6dd02..63d7b7d 100644 --- a/spatialpandas/geometry/base.py +++ b/spatialpandas/geometry/base.py @@ -219,7 +219,7 @@ def __init__(self, array, dtype=None, copy=None): try: if len(array) == 0 and dtype is None: dtype = 'float64' - except: + except Exception: # len failed pass diff --git a/spatialpandas/spatialindex/hilbert_curve.py b/spatialpandas/spatialindex/hilbert_curve.py index d9b01c4..d2ae904 100644 --- a/spatialpandas/spatialindex/hilbert_curve.py +++ b/spatialpandas/spatialindex/hilbert_curve.py @@ -1,12 +1,13 @@ -import numpy as np - -from ..utils import ngjit - """ Initially based on https://github.com/galtay/hilbert_curve, but specialized for 2 dimensions with numba acceleration """ +import numpy as np + +from ..utils import ngjit + + @ngjit def _int_2_binary(v, width): diff --git a/spatialpandas/tests/geometry/__init__.py b/spatialpandas/tests/geometry/__init__.py index e69de29..f148f0b 100644 --- a/spatialpandas/tests/geometry/__init__.py +++ b/spatialpandas/tests/geometry/__init__.py @@ -0,0 +1,4 @@ +import pytest + +pytest.importorskip("geopandas") +pytest.importorskip("shapely") diff --git a/spatialpandas/tests/geometry/algorithms/test_intersection.py b/spatialpandas/tests/geometry/algorithms/test_intersection.py index c52e712..7a37827 100644 --- a/spatialpandas/tests/geometry/algorithms/test_intersection.py +++ b/spatialpandas/tests/geometry/algorithms/test_intersection.py @@ -62,7 +62,10 @@ def test_segment_intersection(ax0, ay0, ax1, ay1, bx0, by0, bx1, by1): assert result1 == result2 # Use shapely polygon to compute expected intersection - if (ax0 == ax1 == bx0 == bx1) or (ay0 == ay1 == by0 == by1): + if ( + (ax0 == ax1 == bx0 == bx1) or (ay0 == ay1 == by0 == by1) + or (ax0 == ax1 == ay0 == ay1) or (bx0 == bx1 == by0 == by1) + ): return line1 = sg.LineString([(ax0, ay0), (ax1, ay1)]) line2 = sg.LineString([(bx0, by0), (bx1, by1)]) diff --git a/spatialpandas/tests/geometry/strategies.py b/spatialpandas/tests/geometry/strategies.py index fb0b723..ff52061 100644 --- a/spatialpandas/tests/geometry/strategies.py +++ b/spatialpandas/tests/geometry/strategies.py @@ -11,7 +11,7 @@ hyp_settings = settings( deadline=None, - max_examples=500, + max_examples=100, suppress_health_check=[HealthCheck.too_slow], ) diff --git a/spatialpandas/tests/geometry/test_geometry.py b/spatialpandas/tests/geometry/test_geometry.py index 0f231af..648b122 100644 --- a/spatialpandas/tests/geometry/test_geometry.py +++ b/spatialpandas/tests/geometry/test_geometry.py @@ -1,4 +1,5 @@ import numpy as np +import pytest from spatialpandas.geometry import ( Line, @@ -109,6 +110,7 @@ def test_polygon_array(): assert polygons.total_bounds == (0.0, 0.0, 3.0, 3.0) def test_polygon_array_from_exterior_coords(): + pytest.importorskip("shapely") from shapely import Polygon p = Polygon(sample_polygon_exterior) @@ -142,7 +144,9 @@ def test_multipolygon_array(): np.testing.assert_equal(multipolygon.area, [17.0, 9.0]) assert multipolygon.total_bounds == (0.0, 0.0, 11.0, 11.0) + def test_multipolygon_array_from_exterior_coords(): + pytest.importorskip("shapely") from shapely import Polygon, MultiPolygon p = Polygon(sample_polygon_exterior) mp = MultiPolygon([p, p]) diff --git a/spatialpandas/tests/test_geodataframe.py b/spatialpandas/tests/test_geodataframe.py index 648bda8..1b5a6c6 100644 --- a/spatialpandas/tests/test_geodataframe.py +++ b/spatialpandas/tests/test_geodataframe.py @@ -2,14 +2,15 @@ import dask import dask.dataframe as dd -import geopandas as gp import pandas as pd import pytest -import shapely.geometry as sg import spatialpandas as sp from spatialpandas import GeoDataFrame, GeoSeries +gp = pytest.importorskip("geopandas") +sg = pytest.importorskip("shapely.geometry") + dask.config.set(scheduler="single-threaded") diff --git a/spatialpandas/tests/test_parquet_s3.py b/spatialpandas/tests/test_parquet_s3.py index c5c8b8b..616dc85 100644 --- a/spatialpandas/tests/test_parquet_s3.py +++ b/spatialpandas/tests/test_parquet_s3.py @@ -1,8 +1,5 @@ import logging import os -import shlex -import subprocess -import time import dask.dataframe as dd import numpy as np @@ -16,10 +13,15 @@ pytest.importorskip("moto") geopandas = pytest.importorskip("geopandas") s3fs = pytest.importorskip("s3fs") -requests = pytest.importorskip("requests") logging.getLogger("botocore").setLevel(logging.INFO) +pytestmark = pytest.mark.xdist_group("s3") + +PORT = 5555 +ENDPOINT_URL = f"http://127.0.0.1:{PORT}/" +BUCKET_NAME = "test_bucket" + @pytest.fixture(scope="module", autouse=True) def s3_fixture(): @@ -27,6 +29,11 @@ def s3_fixture(): Taken from `universal_pathlib/upath/tests` and `s3fs/tests/test_s3fs.py`. """ + from moto.moto_server.threaded_moto_server import ThreadedMotoServer + + server = ThreadedMotoServer(ip_address="127.0.0.1", port=PORT) + server.start() + if "BOTO_CONFIG" not in os.environ: # pragma: no cover os.environ["BOTO_CONFIG"] = "/dev/null" if "AWS_ACCESS_KEY_ID" not in os.environ: # pragma: no cover @@ -40,37 +47,12 @@ def s3_fixture(): if "AWS_DEFAULT_REGION" not in os.environ: # pragma: no cover os.environ["AWS_DEFAULT_REGION"] = "us-east-1" - port = 5555 - bucket_name = "test_bucket" - endpoint_url = f"http://127.0.0.1:{port}/" - proc = subprocess.Popen( - shlex.split(f"moto_server s3 -p {port}"), - stderr=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - ) - try: - timeout = 5 - while timeout > 0: - try: - r = requests.get(endpoint_url, timeout=10) - if r.ok: - break - except Exception: # pragma: no cover - pass - timeout -= 0.1 # pragma: no cover - time.sleep(0.1) # pragma: no cover - anon = False - s3so = { - "anon": anon, - "endpoint_url": endpoint_url, - } - fs = s3fs.S3FileSystem(**s3so) - fs.mkdir(bucket_name) - assert fs.exists(bucket_name) - yield f"s3://{bucket_name}", s3so - finally: - proc.terminate() - proc.wait() + s3so = {"anon": False, "endpoint_url": ENDPOINT_URL} + fs = s3fs.S3FileSystem(**s3so) + fs.mkdir(BUCKET_NAME) + assert fs.exists(BUCKET_NAME) + yield f"s3://{BUCKET_NAME}", s3so + server.stop() @pytest.fixture(scope="module") @@ -101,33 +83,18 @@ def s3_parquet_pandas(s3_fixture, sdf): yield path, s3so, sdf -class TestS3ParquetDask: - @staticmethod - def test_read_parquet_dask_remote_glob_parquet(s3_parquet_dask): - path, s3so, sdf = s3_parquet_dask - result = read_parquet_dask(f"{path}/*.parquet", storage_options=s3so).compute() - assert result.equals(sdf) - - @staticmethod - def test_read_parquet_dask_remote_glob_all(s3_parquet_dask): - path, s3so, sdf = s3_parquet_dask - result = read_parquet_dask(f"{path}/*", storage_options=s3so).compute() - assert result.equals(sdf) - - @staticmethod - def test_read_parquet_dask_remote_dir(s3_parquet_dask): - path, s3so, sdf = s3_parquet_dask - result = read_parquet_dask(path, storage_options=s3so).compute() - assert result.equals(sdf) - - @staticmethod - def test_read_parquet_dask_remote_dir_slash(s3_parquet_dask): - path, s3so, sdf = s3_parquet_dask - result = read_parquet_dask(f"{path}/", storage_options=s3so).compute() - assert result.equals(sdf) +@pytest.mark.parametrize( + "fpath", + ["{path}/*.parquet", "{path}/*", "{path}", "{path}/"], + ids=["glob_parquet", "glob_all", "dir", "dir_slash"], +) +def test_read_parquet_dask_remote(s3_parquet_dask, fpath): + path, s3so, sdf = s3_parquet_dask + result = read_parquet_dask(fpath.format(path=path), storage_options=s3so).compute() + assert result.equals(sdf) -def test_read_parquet_remote(s3_parquet_pandas): +def test_read_parquet_pandas_remote(s3_parquet_pandas): path, s3so, sdf = s3_parquet_pandas result = read_parquet(path, storage_options=s3so) assert result.equals(sdf) diff --git a/spatialpandas/tests/tools/test_sjoin.py b/spatialpandas/tests/tools/test_sjoin.py index 44ff4d3..f83dac1 100644 --- a/spatialpandas/tests/tools/test_sjoin.py +++ b/spatialpandas/tests/tools/test_sjoin.py @@ -1,29 +1,17 @@ import dask.dataframe as dd -import geopandas as gp import numpy as np import pandas as pd import pytest from hypothesis import given import spatialpandas as sp -from ..geometry.strategies import st_point_array, st_polygon_array from ..test_parquet import hyp_settings from spatialpandas import GeoDataFrame from spatialpandas.dask import DaskGeoDataFrame -try: - from geopandas._compat import HAS_RTREE, USE_PYGEOS - gpd_spatialindex = USE_PYGEOS or HAS_RTREE -except ImportError: - try: - import rtree # noqa - gpd_spatialindex = rtree - except Exception: - gpd_spatialindex = False - -if not gpd_spatialindex: - pytest.skip('Geopandas spatialindex not available to compare against', - allow_module_level=True) +gp = pytest.importorskip("geopandas") +rtree = pytest.importorskip("rtree") +from ..geometry.strategies import st_point_array, st_polygon_array # noqa: E402 @pytest.mark.slow diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 46052d4..0000000 --- a/tox.ini +++ /dev/null @@ -1,54 +0,0 @@ -# For use with pyct (https://github.com/holoviz-dev/pyct), but just standard -# tox config (works with tox alone). - -[tox] -# python version test group extra envs extra commands -envlist = {py39,py310,py311}-{lint,unit,unit_deploy,all}-{default}-{dev,pkg} -build = wheel - -[_lint] -description = Flake check python and notebooks, and verify notebooks -deps = .[tests] -# verify takes quite a long time - maybe split into flakes and lint? -commands = flake8 - -[_unit] -description = Run unit tests -deps = .[tests] -commands = pytest spatialpandas --cov=./ - -[_unit_deploy] -description = Run unit tests without coverage -deps = .[tests] -commands = pytest spatialpandas - -[_all] -description = Run all tests (but only including default examples) -deps = .[examples, tests] -commands = {[_lint]commands} - {[_unit_deploy]commands} - -[testenv] -changedir = {envtmpdir} - -commands = unit: {[_unit]commands} - unit_deploy: {[_unit_deploy]commands} - lint: {[_lint]commands} - all: {[_all]commands} - -deps = unit: {[_unit]deps} - unit_deploy: {[_unit_deploy]deps} - lint: {[_lint]deps} - all: {[_all]deps} - -[pytest] -addopts = -v --pyargs --doctest-modules --doctest-ignore-import-errors --color=yes -norecursedirs = doc .git dist build _build .ipynb_checkpoints - -[flake8] -include = *.py -# run_tests.py is generated by conda build, which appears to have a -# bug resulting in code being duplicated a couple of times. -exclude = .git,__pycache__,.tox,.eggs,*.egg,doc,dist,build,_build,.ipynb_checkpoints,run_test.py -ignore = E, - W