Merge branch 'main' into test

.circleci/config.yml

@@ -297,6 +297,26 @@ jobs:
           paths:
             - ~/mne_data/eeg_matchingpennies
 
+  cache_MNE-phantom-KIT-data:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      - restore_cache:
+          keys:
+            - data-cache-MNE-phantom-KIT-data-1
+      - bash_env
+      - gitconfig  # email address is needed for datalad
+      - run:
+          name: Get MNE-phantom-KIT-data
+          command: |
+            $DOWNLOAD_DATA MNE-phantom-KIT-data
+      - codecov/upload
+      - save_cache:
+          key: data-cache-MNE-phantom-KIT-data-1
+          paths:
+            - ~/mne_data/MNE-phantom-KIT-data
+
   cache_ERP_CORE:
     <<: *imageconfig
     steps:
@@ -765,6 +785,32 @@ jobs:
           paths:
             - mne_data/derivatives/mne-bids-pipeline/eeg_matchingpennies/*/*/*.html
 
+  test_MNE-phantom-KIT-data:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      - bash_env
+      - restore_cache:
+          keys:
+            - data-cache-MNE-phantom-KIT-data-1
+      - run:
+          name: test MNE-phantom-KIT-data
+          command: $RUN_TESTS MNE-phantom-KIT-data
+      - codecov/upload
+      - store_test_results:
+          path: ./test-results
+      - store_artifacts:
+          path: ./test-results
+          destination: test-results
+      - store_artifacts:
+          path: /home/circleci/reports/MNE-phantom-KIT-data
+          destination: reports/MNE-phantom-KIT-data
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - mne_data/derivatives/mne-bids-pipeline/MNE-phantom-KIT-data/*/*/*.html
+
   test_ERP_CORE_N400:
     <<: *imageconfig
     resource_class: large
@@ -1191,6 +1237,15 @@ workflows:
             - cache_eeg_matchingpennies
           <<: *filter_tags
 
+      - cache_MNE-phantom-KIT-data:
+          requires:
+            - setup_env
+          <<: *filter_tags
+      - test_MNE-phantom-KIT-data:
+          requires:
+            - cache_MNE-phantom-KIT-data
+          <<: *filter_tags
+
       - cache_ERP_CORE:
           requires:
             - setup_env
@@ -1242,6 +1297,7 @@ workflows:
             - test_ds003392
             - test_ds004229
             - test_eeg_matchingpennies
+            - test_MNE-phantom-KIT-data
             - test_ERP_CORE_N400
             - test_ERP_CORE_ERN
             - test_ERP_CORE_LRP

.github/workflows/run-tests.yml

@@ -6,32 +6,70 @@ concurrency:
 on: [push, pull_request]
 
 jobs:
-  check-style:
-    name: Style
-    runs-on: "ubuntu-latest"
-    defaults:
-      run:
-        shell: bash -l {0}
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-    - name: Install ruff and codespell
-      run: pip install ruff codespell tomli
-    - run: make ruff
-    - run: make codespell-error
-    - uses: psf/black@stable
   check-doc:
-    name: Doc consistency
+    name: Doc consistency and codespell
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
     - run: pip install --upgrade pip
-    - run: pip install -ve .[tests]
+    - run: pip install -ve .[tests] codespell tomli
+    - run: make codespell-error
     - run: pytest mne_bids_pipeline -m "not dataset_test"
     - uses: codecov/codecov-action@v3
       if: success()
       name: 'Upload coverage to CodeCov'
+  caching:
+    name: 'Caching on ${{ matrix.os }}'
+    timeout-minutes: 30
+    continue-on-error: true
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+          - os: windows-latest
+    env:
+      MNE_BIDS_PIPELINE_LEGACY_WINDOWS: "false"
+      PYTHONIOENCODING: 'utf8'  # for Windows
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pyvista/setup-headless-display-action@main
+        with:
+          qt: true
+          pyvista: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"  # no "multidict" wheels on 3.12 yet
+      - run: pip install -ve .[tests]
+      - uses: actions/cache@v4
+        with:
+          key: ds001971
+          path: ~/mne_data/ds001971
+        id: ds001971-cache
+      - run: python -m mne_bids_pipeline._download ds001971
+        if: steps.ds001971-cache.outputs.cache-hit != 'true'
+      - run: pytest --cov-append -k ds001971 mne_bids_pipeline/
+      - run: pytest --cov-append -k ds001971 mne_bids_pipeline/  # uses "hash" method
+        timeout-minutes: 1
+      - uses: actions/cache@v4
+        with:
+          key: ds003392
+          path: ~/mne_data/ds003392
+        id: ds003392-cache
+      - run: python -m mne_bids_pipeline._download ds003392
+        if: steps.ds003392-cache.outputs.cache-hit != 'true'
+      - run: pytest --cov-append -k ds003392 mne_bids_pipeline/
+      - run: pytest --cov-append -k ds003392 mne_bids_pipeline/  # uses "mtime" method
+        timeout-minutes: 1
+      - uses: codecov/codecov-action@v3
+        if: success()

.pre-commit-config.yaml

@@ -5,17 +5,12 @@ files: ^(.*\.(py|yaml))$
 # for example
 exclude: ^(\.[^/]*cache/.*|.*/freesurfer/contrib/.*)$
 repos:
-  - repo: https://github.com/psf/black
-    rev: 23.12.1
-    hooks:
-      - id: black
-        args:
-          - --safe
-          - --quiet
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.13
+    rev: v0.2.0
     hooks:
       - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:

Makefile

@@ -33,10 +33,6 @@ check:
 trailing-spaces:
 	find . -name "*.py" | xargs perl -pi -e 's/[ \t]*$$//'
 
-ruff:
-	ruff .
-	@echo "ruff passed"
-
 codespell:  # running manually; auto-fix spelling mistakes
 	@codespell --write-changes $(CODESPELL_DIRS)
 

docs/hooks.py

@@ -1,9 +1,9 @@
 import logging
-from typing import Dict, Any
+from typing import Any
 
 from mkdocs.config.defaults import MkDocsConfig
-from mkdocs.structure.pages import Page
 from mkdocs.structure.files import Files
+from mkdocs.structure.pages import Page
 
 logger = logging.getLogger("mkdocs")
 
@@ -13,7 +13,7 @@
 # Ideally there would be a better hook, but it's unclear if context can
 # be obtained any earlier
 def on_template_context(
-    context: Dict[str, Any],
+    context: dict[str, Any],
     template_name: str,
     config: MkDocsConfig,
 ) -> None:

docs/mkdocs.yml

@@ -90,7 +90,7 @@ nav:
             - Epoching: settings/preprocessing/epochs.md
             - Artifact removal:
                 - Stimulation artifact: settings/preprocessing/stim_artifact.md
-                - SSP & ICA: settings/preprocessing/ssp_ica.md
+                - SSP, ICA, and artifact regression: settings/preprocessing/ssp_ica.md
                 - Amplitude-based artifact rejection: settings/preprocessing/artifacts.md
         - Sensor-level analysis:
             - Condition contrasts: settings/sensor/contrasts.md
@@ -116,6 +116,7 @@ nav:
         - examples/ds000248_no_mri.md
         - examples/ds003104.md
         - examples/eeg_matchingpennies.md
+        - examples/MNE-phantom-KIT-data.md
         - examples/ds001810.md
         - examples/ds000117.md
         - examples/ds003775.md

docs/source/examples/gen_examples.py

@@ -1,19 +1,21 @@
 #!/usr/bin/env python
 
-from collections import defaultdict
 import contextlib
 import logging
 import shutil
-from pathlib import Path
 import sys
-from typing import Union, Iterable
+from collections import defaultdict
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Union
+
+from tqdm import tqdm
 
 import mne_bids_pipeline
-from mne_bids_pipeline._config_import import _import_config
 import mne_bids_pipeline.tests.datasets
-from mne_bids_pipeline.tests.test_run import TEST_SUITE
+from mne_bids_pipeline._config_import import _import_config
 from mne_bids_pipeline.tests.datasets import DATASET_OPTIONS
-from tqdm import tqdm
+from mne_bids_pipeline.tests.test_run import TEST_SUITE
 
 this_dir = Path(__file__).parent
 root = Path(mne_bids_pipeline.__file__).parent.resolve(strict=True)
@@ -61,6 +63,8 @@ def _gen_demonstrated_funcs(example_config_path: Path) -> dict:
         key = "Maxwell filter"
         funcs[key] = funcs[key] or config.use_maxwell_filter
         funcs["Frequency filter"] = config.l_freq or config.h_freq
+        key = "Artifact regression"
+        funcs[key] = funcs[key] or (config.regress_artifact is not None)
         key = "SSP"
         funcs[key] = funcs[key] or (config.spatial_filter == "ssp")
         key = "ICA"
@@ -142,6 +146,7 @@ def _gen_demonstrated_funcs(example_config_path: Path) -> dict:
         logger.warning(f"Dataset {dataset_name} has no HTML report.")
         continue
 
+    assert dataset_options_key in DATASET_OPTIONS, dataset_options_key
     options = DATASET_OPTIONS[dataset_options_key].copy()  # we modify locally
 
     report_str = "\n## Generated output\n\n"
@@ -198,13 +203,18 @@ def _gen_demonstrated_funcs(example_config_path: Path) -> dict:
             f"{fname.name} :fontawesome-solid-square-poll-vertical:</a>\n\n"
         )
 
-    assert sum(key in options for key in ("openneuro", "git", "web", "datalad")) == 1
+    assert (
+        sum(key in options for key in ("openneuro", "git", "web", "datalad", "mne"))
+        == 1
+    )
     if "openneuro" in options:
         url = f'https://openneuro.org/datasets/{options["openneuro"]}'
     elif "git" in options:
         url = options["git"]
     elif "web" in options:
         url = options["web"]
+    elif "mne" in options:
+        url = f"https://mne.tools/dev/generated/mne.datasets.{options['mne']}.data_path.html"  # noqa: E501
     else:
         assert "datalad" in options  # guaranteed above
         url = ""
@@ -244,7 +254,9 @@ def _gen_demonstrated_funcs(example_config_path: Path) -> dict:
 
     # TODO: For things like ERP_CORE_ERN, decoding_csp are not populated
     # properly by the root config
-    config_path = root / "tests" / "configs" / f"config_{dataset_name}.py"
+    config_path = (
+        root / "tests" / "configs" / f"config_{dataset_name.replace('-', '_')}.py"
+    )
     config = config_path.read_text(encoding="utf-8-sig").strip()
     descr_end_idx = config[2:].find('"""')
     config_descr = "# " + config[: descr_end_idx + 1].replace('"""', "").strip()

docs/source/features/gen_steps.py

@@ -3,6 +3,7 @@
 
 import importlib
 from pathlib import Path
+
 from mne_bids_pipeline._config_utils import _get_step_modules
 
 pre = """\

docs/source/settings/preprocessing/ssp_ica.md

@@ -11,6 +11,7 @@ tags:
 ::: mne_bids_pipeline._config
     options:
       members:
+        - regress_artifact
         - spatial_filter
         - min_ecg_epochs
         - min_eog_epochs

docs/source/v1.6.md.inc

@@ -2,9 +2,12 @@
 
 ## vX.Y.0 (unreleased)
 
-[//]: # (### :new: New features & enhancements)
+:new: New features & enhancements
 
-[//]: # (- Whatever (#000 by @whoever))
+- Added [`regress_artifact`][mne_bids_pipeline._config.regress_artifact] to allow artifact regression (e.g., of MEG reference sensors in KIT systems) (#837 by @larsoner)
+- Chosen `reject` parameters are now saved in the generated HTML reports (#839 by @larsoner)
+- Added saving of clean raw data in addition to epochs (#840 by @larsoner)
+- Added saving of detected blink and cardiac events used to calculate SSP projectors (#840 by @larsoner)
 
 [//]: # (### :warning: Behavior changes)
 
@@ -14,10 +17,17 @@
 
 - MNE-BIDS-Pipeline now requires Python 3.9 or newer. (#825 by @hoechenberger)
 
-[//]: # (### :bug: Bug fixes)
+### :bug: Bug fixes
 
-[//]: # (- Whatever (#000 by @whoever))
+- Fix minor issues with path handling for cross-talk and calibration files (#834 by @larsoner)
+- Fix bug where EEG `reject` params were not used for `ch_types = ["meg", "eeg"]` (#839 by @larsoner)
+- Fix bug where implicit `mf_reference_run` could change across invocations of `mne_bids_pipeline`, breaking caching (#839 by @larsoner)
+- Fix bug where `--no-cache` had no effect (#839 by @larsoner)
+- Fix bug where the Maxwell filtering step would fail if [`find_noisy_channels_meg = False`][mne_bids_pipeline._config.find_noisy_channels_meg]` was used (#847 by @larsoner)
+- Fix bug where raw, empty-room, and custom noise covariances were errantly calculated on data without ICA or SSP applied (#840 by @larsoner)
 
 ### :medical_symbol: Code health
 
 - The package build backend has been switched from `setuptools` to `hatchling`. (#825 by @hoechenberger)
+- Code formatting now uses `ruff format` instead of `black` (#834, #838 by @larsoner)
+- Code caching is now tested using GitHub Actions (#836 by @larsoner)

mne_bids_pipeline/__init__.py

@@ -1,4 +1,4 @@
-from importlib.metadata import version, PackageNotFoundError
+from importlib.metadata import PackageNotFoundError, version
 
 try:
     __version__ = version("mne_bids_pipeline")