From 57d7ab952438b944eff2913c6d72a8b1fd1dd993 Mon Sep 17 00:00:00 2001 From: Nate Parsons <4307001+thehomebrewnerd@users.noreply.github.com> Date: Fri, 9 Feb 2024 14:55:06 -0600 Subject: [PATCH] Add Dask to latest and min dependency checkers (#2658) * add dask to latest dependency checker * add dask to latest dependency checker * update min deps * update release notes * add dask min reqs file * fix min tests with spark --- .../workflows/latest_dependency_checker.yaml | 2 +- .../workflows/minimum_dependency_checker.yaml | 8 ++++++++ .github/workflows/tests_with_minimum_deps.yaml | 17 +++++++++++++++-- docs/source/release_notes.rst | 1 + .../minimum_dask_requirements.txt | 11 +++++++++++ 5 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 featuretools/tests/requirement_files/minimum_dask_requirements.txt diff --git a/.github/workflows/latest_dependency_checker.yaml b/.github/workflows/latest_dependency_checker.yaml index 1c28906450..6fda3a296d 100644 --- a/.github/workflows/latest_dependency_checker.yaml +++ b/.github/workflows/latest_dependency_checker.yaml @@ -23,7 +23,7 @@ jobs: - name: Update dependencies run: | python -m pip install --upgrade pip - python -m pip install -e ".[spark,test]" + python -m pip install -e ".[dask,spark,test]" make checkdeps OUTPUT_PATH=featuretools/tests/requirement_files/latest_requirements.txt - name: Create pull request uses: peter-evans/create-pull-request@v3 diff --git a/.github/workflows/minimum_dependency_checker.yaml b/.github/workflows/minimum_dependency_checker.yaml index 4da5a85579..0a74157b6d 100644 --- a/.github/workflows/minimum_dependency_checker.yaml +++ b/.github/workflows/minimum_dependency_checker.yaml @@ -30,6 +30,14 @@ jobs: paths: 'pyproject.toml' options: 'dependencies' output_filepath: featuretools/tests/requirement_files/minimum_core_requirements.txt + - name: Run min dep generator - dask + id: min_dep_gen_dask + uses: alteryx/minimum-dependency-generator@v3 + with: + paths: 'pyproject.toml' + options: 'dependencies' + extras_require: 'dask' + output_filepath: featuretools/tests/requirement_files/minimum_dask_requirements.txt - name: Run min dep generator - spark id: min_dep_gen_spark uses: alteryx/minimum-dependency-generator@v3 diff --git a/.github/workflows/tests_with_minimum_deps.yaml b/.github/workflows/tests_with_minimum_deps.yaml index 0d16848599..9ba64277f2 100644 --- a/.github/workflows/tests_with_minimum_deps.yaml +++ b/.github/workflows/tests_with_minimum_deps.yaml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - libraries: ["core", "spark - misc", "spark - computational", "spark - entityset_1", "spark - entityset_2", "spark - primitives"] + libraries: ["core", "dask", "spark - misc", "spark - computational", "spark - entityset_1", "spark - entityset_2", "spark - primitives"] steps: - name: Checkout repository uses: actions/checkout@v3 @@ -45,10 +45,16 @@ jobs: NUMPY_VERSION=$(cat featuretools/tests/requirement_files/minimum_core_requirements.txt | grep numpy) python -m pip uninstall numpy -y python -m pip install $NUMPY_VERSION --no-build-isolation + - if: ${{ matrix.libraries == 'dask' }} + name: Install numpy for dask + run: | + NUMPY_VERSION=$(cat featuretools/tests/requirement_files/minimum_dask_requirements.txt | grep numpy) + python -m pip uninstall numpy -y + python -m pip install $NUMPY_VERSION --no-build-isolation - name: Install featuretools - minimum tests dependencies run: | python -m pip install -r featuretools/tests/requirement_files/minimum_test_requirements.txt - - if: ${{ matrix.libraries == 'spark' }} + - if: ${{ startsWith(matrix.libraries, 'spark') }} name: Install featuretools - minimum spark, core dependencies run: | sudo apt install -y openjdk-11-jre-headless @@ -58,9 +64,16 @@ jobs: name: Install featuretools - minimum core dependencies run: | python -m pip install -r featuretools/tests/requirement_files/minimum_core_requirements.txt + - if: ${{ matrix.libraries == 'dask' }} + name: Install featuretools - minimum dask dependencies + run: | + python -m pip install -r featuretools/tests/requirement_files/minimum_dask_requirements.txt - if: ${{ matrix.libraries == 'core' }} name: Run unit tests without code coverage run: python -m pytest -x -n auto featuretools/tests/ + - if: ${{ matrix.libraries == 'dask' }} + name: Run dask unit tests without code coverage + run: python -m pytest -x -n auto featuretools/tests/ - if: ${{ matrix.libraries == 'spark - misc' }} name: Run unit tests (misc) run: pytest featuretools/ -n auto --ignore=featuretools/tests/computational_backend --ignore=featuretools/tests/entityset_tests --ignore=featuretools/tests/primitive_tests diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 88e8f23f8a..928465bf0b 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -14,6 +14,7 @@ Future Release * Update tests for compatibility with new versions of ``holidays`` (:pr:`2636`) * Update ruff to 0.1.6 and use ruff linter/formatter (:pr:`2639`) * Update ``release.yaml`` to use trusted publisher for PyPI releases (:pr:`2646`, :pr:`2653`, :pr:`2654`) + * Update dependency checkers and tests to include Dask (:pr:`2658`) Thanks to the following people for contributing to this release: diff --git a/featuretools/tests/requirement_files/minimum_dask_requirements.txt b/featuretools/tests/requirement_files/minimum_dask_requirements.txt new file mode 100644 index 0000000000..c7d8827fd4 --- /dev/null +++ b/featuretools/tests/requirement_files/minimum_dask_requirements.txt @@ -0,0 +1,11 @@ +cloudpickle==1.5.0 +holidays==0.17 +numpy==1.21.0 +packaging==20.0 +pandas==1.5.0 +psutil==5.6.6 +scipy==1.10.0 +tqdm==4.32.0 +woodwork[dask]==0.23.0 +dask[dataframe]==2022.11.1 +distributed==2022.11.1