Add machinery to run test only on changed files #54
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Complete tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 12 * * 0" # Weekly on Sunday at noon UTC | |
| pull_request: | |
| types: [synchronize, opened, reopened] | |
| branches: | |
| - main | |
| env: | |
| KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }} | |
| KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }} | |
| concurrency: # Cancel previous workflows on the same pull request | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run: | |
| name: ${{ matrix.os }} Python ${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.9", "3.12"] # Lower and higher versions we support | |
| os: [macos-13, windows-latest, ubuntu-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| # cache: 'pip' # caching pip dependencies | |
| - name: Get current hash (SHA) of the ephy_testing_data repo | |
| id: repo_hash | |
| run: echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Cache datasets | |
| id: cache-datasets | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/spikeinterface_datasets | |
| key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }} | |
| restore-keys: ${{ runner.os }}-datasets | |
| - name: Install packages | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "CI Almighty" | |
| pip install -e .[test,extractors,streaming_extractors,full] | |
| pip install tabulate | |
| shell: bash | |
| - name: Installad datalad | |
| run: | | |
| pip install datalad-installer | |
| if [ ${{ runner.os }} = 'Linux' ]; then | |
| datalad-installer --sudo ok git-annex --method datalad/packages | |
| elif [ ${{ runner.os }} = 'macOS' ]; then | |
| datalad-installer --sudo ok git-annex --method brew | |
| elif [ ${{ runner.os }} = 'Windows' ]; then | |
| datalad-installer --sudo ok git-annex --method datalad/git-annex:release | |
| fi | |
| pip install datalad | |
| git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency | |
| shell: bash | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/changed-files@v41 | |
| - name: Set environment variables based on changed files | |
| run: | | |
| declare -A changes | |
| changes=( | |
| ["pyproject.toml"]="CORE_CHANGED" | |
| ["/core/"]="CORE_CHANGED" | |
| ["/extractors/neoextractors/neobaseextractor.py"]="CORE_CHANGED" | |
| ["/extractors/"]="EXTRACTORS_CHANGED" | |
| ["plexon2"]="PLEXON2_CHANGED" | |
| ["/preprocessing/"]="PREPROCESSING_CHANGED" | |
| ["/postprocessing/"]="POSTPROCESSING_CHANGED" | |
| ["/qualitymetrics/"]="QUALITYMETRICS_CHANGED" | |
| ["/sorters/"]="SORTERS_CHANGED" | |
| ["/sorters/external"]="SORTERS_EXTERNAL_CHANGED" | |
| ["/sorters/internal"]="SORTERS_INTERNAL_CHANGED" | |
| ["/comparison/"]="COMPARISON_CHANGED" | |
| ["/curation/"]="CURATION_CHANGED" | |
| ["/widgets/"]="WIDGETS_CHANGED" | |
| ["/exporters/"]="EXPORTERS_CHANGED" | |
| ["/sortingcomponents/"]="SORTINGCOMPONENTS_CHANGED" | |
| ["/generation/"]="GENERATION_CHANGED" | |
| ) | |
| for file in ${{ steps.changed-files.outputs.all_changed_files }}; do | |
| for pattern in "${!changes[@]}"; do | |
| if [[ $file == *$pattern* ]]; then | |
| echo "${changes[$pattern]}=true" >> $GITHUB_ENV | |
| fi | |
| done | |
| done | |
| - name: Set execute permissions on run_tests.sh | |
| run: chmod +x .github/run_tests.sh | |
| shell: bash | |
| - name: Test core | |
| run: pytest -m "core" | |
| shell: bash | |
| - name: Test extractors | |
| env: | |
| HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_plugin_path_maxwell | |
| if: env.EXTRACTORS_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: pytest -m "extractors" | |
| shell: bash | |
| - name: Test preprocessing | |
| if: env.PREPROCESSING_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh "preprocessing and not deepinterpolation" --no-virtual-env | |
| shell: bash | |
| - name: Test postprocessing | |
| if: env.POSTPROCESSING_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh postprocessing --no-virtual-env | |
| shell: bash | |
| - name: Test quality metrics | |
| if: env.QUALITYMETRICS_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh qualitymetrics --no-virtual-env | |
| shell: bash | |
| - name: Test comparison | |
| if: env.COMPARISON_CHANGED == 'true' || env.GENERATION_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh comparison --no-virtual-env | |
| shell: bash | |
| - name: Test core sorters | |
| if: env.SORTERS_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh sorters --no-virtual-env | |
| shell: bash | |
| - name: Test internal sorters | |
| if: env.SORTERS_INTERNAL_CHANGED == 'true' || env.SORTINGCOMPONENTS_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh sorters_internal --no-virtual-env | |
| shell: bash | |
| - name: Test curation | |
| if: env.CURATION_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh curation --no-virtual-env | |
| shell: bash | |
| - name: Test widgets | |
| if: env.WIDGETS_CHANGED == 'true' || env.CORE_CHANGED == 'true' || env.QUALITYMETRICS_CHANGED == 'true' || env.PREPROCESSING_CHANGED == 'true' | |
| run: ./.github/run_tests.sh widgets --no-virtual-env | |
| shell: bash | |
| - name: Test exporters | |
| if: env.EXPORTERS_CHANGED == 'true' || env.CORE_CHANGED == 'true' || env.WIDGETS_CHANGED == 'true' | |
| run: ./.github/run_tests.sh exporters --no-virtual-env | |
| shell: bash | |
| - name: Test sortingcomponents | |
| if: env.SORTINGCOMPONENTS_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh sortingcomponents --no-virtual-env | |
| shell: bash | |
| - name: Test generation | |
| if: env.GENERATION_CHANGED == 'true' || env.CORE_CHANGED == 'true' | |
| run: ./.github/run_tests.sh generation --no-virtual-env | |
| shell: bash |