ENH: Cache grand-average steps (#765)

.circleci/config.yml

@@ -82,6 +82,7 @@ jobs:
           name: Get ds000117
           command: |
             $DOWNLOAD_DATA ds000117
+      - codecov/upload
       - save_cache:
           key: data-cache-ds000117-2
           paths:
@@ -118,6 +119,7 @@ jobs:
           name: Get ds001971
           command: |
             $DOWNLOAD_DATA ds001971
+      - codecov/upload
       - save_cache:
           key: data-cache-ds001971-2
           paths:
@@ -136,6 +138,7 @@ jobs:
           name: Get ds004107
           command: |
             $DOWNLOAD_DATA ds004107
+      - codecov/upload
       - save_cache:
           key: data-cache-ds004107-2
           paths:
@@ -154,6 +157,7 @@ jobs:
           name: Get ds000246
           command: |
             $DOWNLOAD_DATA ds000246
+      - codecov/upload
       - save_cache:
           key: data-cache-ds000246-2
           paths:
@@ -172,6 +176,7 @@ jobs:
           name: Get ds000247
           command: |
             $DOWNLOAD_DATA ds000247
+      - codecov/upload
       - save_cache:
           key: data-cache-ds000247-2
           paths:
@@ -190,6 +195,7 @@ jobs:
           name: Get ds000248
           command: |
             $DOWNLOAD_DATA ds000248
+      - codecov/upload
       - save_cache:
           key: data-cache-ds000248-4
           paths:
@@ -208,6 +214,7 @@ jobs:
           name: Get ds001810
           command: |
             $DOWNLOAD_DATA ds001810
+      - codecov/upload
       - save_cache:
           key: data-cache-ds001810-2
           paths:
@@ -226,6 +233,7 @@ jobs:
           name: Get ds003104
           command: |
             $DOWNLOAD_DATA ds003104
+      - codecov/upload
       - save_cache:
           key: data-cache-ds003104-2
           paths:
@@ -244,6 +252,7 @@ jobs:
           name: Get ds003392
           command: |
             $DOWNLOAD_DATA ds003392
+      - codecov/upload
       - save_cache:
           key: data-cache-ds003392-2
           paths:
@@ -262,6 +271,7 @@ jobs:
           name: Get ds004229
           command: |
             $DOWNLOAD_DATA ds004229
+      - codecov/upload
       - save_cache:
           key: data-cache-ds004229-2
           paths:
@@ -281,6 +291,7 @@ jobs:
           name: Get eeg_matchingpennies
           command: |
             $DOWNLOAD_DATA eeg_matchingpennies
+      - codecov/upload
       - save_cache:
           key: data-cache-eeg_matchingpennies-1
           paths:
@@ -299,6 +310,7 @@ jobs:
           name: Get ERP_CORE
           command: |
             $DOWNLOAD_DATA ERP_CORE
+      - codecov/upload
       - save_cache:
           key: data-cache-ERP_CORE-1
           paths:
@@ -474,7 +486,8 @@ jobs:
             - data-cache-ds000248-4
       - run:
           name: test ds000248_base
-          command: $RUN_TESTS ds000248_base
+          # Forces rerunning (cov and FLASH BEM) so don't check
+          command: $RUN_TESTS -r ds000248_base
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -529,7 +542,7 @@ jobs:
             - data-cache-ds000248-4
       - run:
           name: test BEM from FLASH
-          command: $RUN_TESTS ds000248_FLASH_BEM
+          command: $RUN_TESTS -r ds000248_FLASH_BEM
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -556,7 +569,7 @@ jobs:
       - run:
           name: test BEM from T1 (watershed)
           no_output_timeout: 20m
-          command: $RUN_TESTS ds000248_T1_BEM
+          command: $RUN_TESTS -r ds000248_T1_BEM
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -582,7 +595,7 @@ jobs:
             - data-cache-ds000248-4
       - run:
           name: test head surface creation for MNE coregistration
-          command: $RUN_TESTS ds000248_coreg_surfaces ds000248_coreg_surfaces --no-copy
+          command: $RUN_TESTS -c -r ds000248_coreg_surfaces
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -772,7 +785,7 @@ jobs:
             google-chrome --version
       - run:
           name: test ERP CORE N400
-          command: $RUN_TESTS ERP_CORE_N400 ERP_CORE
+          command: $RUN_TESTS ERP_CORE_N400
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -802,7 +815,7 @@ jobs:
           command: mkdir -p /home/circleci/.local/share/pyvista
       - run:
           name: test ERP CORE ERN
-          command: $RUN_TESTS ERP_CORE_ERN ERP_CORE
+          command: $RUN_TESTS ERP_CORE_ERN
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -832,7 +845,7 @@ jobs:
           command: mkdir -p /home/circleci/.local/share/pyvista
       - run:
           name: test ERP CORE LRP
-          command: $RUN_TESTS ERP_CORE_LRP ERP_CORE
+          command: $RUN_TESTS ERP_CORE_LRP
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -862,7 +875,7 @@ jobs:
           command: mkdir -p /home/circleci/.local/share/pyvista
       - run:
           name: test ERP CORE MMN
-          command: $RUN_TESTS ERP_CORE_MMN ERP_CORE
+          command: $RUN_TESTS ERP_CORE_MMN
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -892,7 +905,7 @@ jobs:
           command: mkdir -p /home/circleci/.local/share/pyvista
       - run:
           name: test ERP CORE N2pc
-          command: $RUN_TESTS ERP_CORE_N2pc ERP_CORE
+          command: $RUN_TESTS ERP_CORE_N2pc
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -922,7 +935,7 @@ jobs:
           command: mkdir -p /home/circleci/.local/share/pyvista
       - run:
           name: test ERP CORE N170
-          command: $RUN_TESTS ERP_CORE_N170 ERP_CORE
+          command: $RUN_TESTS ERP_CORE_N170
       - codecov/upload
       - store_test_results:
           path: ./test-results
@@ -952,7 +965,7 @@ jobs:
           command: mkdir -p /home/circleci/.local/share/pyvista
       - run:
           name: test ERP CORE P3
-          command: $RUN_TESTS ERP_CORE_P3 ERP_CORE
+          command: $RUN_TESTS ERP_CORE_P3
       - codecov/upload
       - store_test_results:
           path: ./test-results

.circleci/run_dataset_and_copy_files.sh

@@ -2,21 +2,48 @@
 
 set -eo pipefail
 
+COPY_FILES="true"
+RERUN_TEST="true"
+while getopts "cr" option; do
+   echo $option
+   case $option in
+      c)
+        COPY_FILES="false";;
+      r)
+        RERUN_TEST="false";;
+   esac
+done
+shift "$(($OPTIND -1))"
+
 DS_RUN=$1
-if [[ "$2" == "" ]]; then
-  DS="$DS_RUN"
-else
-  DS="$2"
+if [[ -z $1 ]]; then
+  echo "Missing dataset argument"
+  exit 1
 fi
-if [[ "$3" == "--no-copy" ]]; then
-  COPY_FILES="false"
+if [[ "$DS_RUN" == "ERP_CORE_"* ]]; then
+  DS="ERP_CORE"
 else
-  COPY_FILES="true"
+  DS="$1"
 fi
 
+SECONDS=0
 pytest mne_bids_pipeline --junit-xml=test-results/junit-results.xml -k ${DS_RUN}
+echo "Runtime: ${SECONDS} seconds"
+
+# rerun test (check caching)!
+SECONDS=0
+if [[ "$RERUN_TEST" == "false" ]]; then
+  echo "Skipping rerun test"
+  RUN_TIME=0
+else
+  pytest mne_bids_pipeline --cov-append -k $DS_RUN
+  RUN_TIME=$SECONDS
+  echo "Runtime: ${RUN_TIME} seconds (should be < 20)"
+fi
+test $RUN_TIME -lt 20
 
 if [[ "$COPY_FILES" == "false" ]]; then
+  echo "Not copying files"
   exit 0
 fi
 mkdir -p ~/reports/${DS}

.circleci/setup_bash.sh

@@ -36,7 +36,7 @@ sudo ln -s /usr/lib/x86_64-linux-gnu/libxcb-util.so.0 /usr/lib/x86_64-linux-gnu/
 wget -q -O- http://neuro.debian.net/lists/focal.us-tn.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
 sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9
 echo "export RUN_TESTS=\".circleci/run_dataset_and_copy_files.sh\"" >> "$BASH_ENV"
-echo "export DOWNLOAD_DATA=\"python -m mne_bids_pipeline._download\"" >> "$BASH_ENV"
+echo "export DOWNLOAD_DATA=\"coverage run -m mne_bids_pipeline._download\"" >> "$BASH_ENV"
 
 # Similar CircleCI setup to mne-python (Xvfb, venv, minimal commands, env vars)
 wget -q https://raw.githubusercontent.com/mne-tools/mne-python/main/tools/setup_xvfb.sh

docs/source/v1.5.md.inc

@@ -6,6 +6,7 @@
 - Added examples of T1 and FLASH BEM to website (#758 by @larsoner)
 - Added support for extended SSS (eSSS) in Maxwell filtering (#762 by @larsoner)
 - Output logging spacing improved (#764 by @larsoner)
+- Added caching of sensor and source average steps (#765 by @larsoner)
 
 [//]: # (### :warning: Behavior changes)
 

mne_bids_pipeline/__init__.py

@@ -2,6 +2,6 @@
 
 try:
     __version__ = version("mne_bids_pipeline")
-except PackageNotFoundError:
+except PackageNotFoundError:  # pragma: no cover
     # package is not installed
     __version__ = "0.0.0"

mne_bids_pipeline/_config.py

@@ -1572,7 +1572,7 @@
 
 time_frequency_subtract_evoked: bool = False
 """
-Whether to subtract the evoked signal (averaged across all epochs) from the
+Whether to subtract the evoked response (averaged across all epochs) from the
 epochs before passing them to time-frequency analysis. Set this to `True` to
 highlight induced activity.
 

mne_bids_pipeline/_config_import.py

@@ -36,6 +36,17 @@ def _import_config(
         log=log,
     )
 
+    extra_exec_params_keys = ()
+    extra_config = os.getenv("_MNE_BIDS_STUDY_TESTING_EXTRA_CONFIG", "")
+    if extra_config:
+        msg = f"With testing config: {extra_config}"
+        logger.info(**gen_log_kwargs(message=msg, emoji="override"))
+        _update_config_from_path(
+            config=config,
+            config_path=extra_config,
+        )
+        extra_exec_params_keys = ("_n_jobs",)
+
     # Check it
     if check:
         _check_config(config)
@@ -69,7 +80,7 @@ def _import_config(
         # Misc
         "deriv_root",
         "config_path",
-    )
+    ) + extra_exec_params_keys
     in_both = {"deriv_root"}
     exec_params = SimpleNamespace(**{k: getattr(config, k) for k in keys})
     for k in keys:
@@ -102,6 +113,32 @@ def _get_default_config():
     return config
 
 
+def _update_config_from_path(
+    *,
+    config: SimpleNamespace,
+    config_path: PathLike,
+):
+    user_names = list()
+    config_path = pathlib.Path(config_path).expanduser().resolve(strict=True)
+    # Import configuration from an arbitrary path without having to fiddle
+    # with `sys.path`.
+    spec = importlib.util.spec_from_file_location(
+        name="custom_config", location=config_path
+    )
+    custom_cfg = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(custom_cfg)
+    for key in dir(custom_cfg):
+        if not key.startswith("__"):
+            # don't validate private vars, but do add to config
+            # (e.g., so that our hidden _raw_split_size is included)
+            if not key.startswith("_"):
+                user_names.append(key)
+            val = getattr(custom_cfg, key)
+            logger.debug("Overwriting: %s -> %s" % (key, val))
+            setattr(config, key, val)
+    return user_names
+
+
 def _update_with_user_config(
     *,
     config: SimpleNamespace,  # modified in-place
@@ -121,23 +158,12 @@ def _update_with_user_config(
     # 2. User config
     user_names = list()
     if config_path is not None:
-        config_path = pathlib.Path(config_path).expanduser().resolve(strict=True)
-        # Import configuration from an arbitrary path without having to fiddle
-        # with `sys.path`.
-        spec = importlib.util.spec_from_file_location(
-            name="custom_config", location=config_path
+        user_names.extend(
+            _update_config_from_path(
+                config=config,
+                config_path=config_path,
+            )
         )
-        custom_cfg = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(custom_cfg)
-        for key in dir(custom_cfg):
-            if not key.startswith("__"):
-                # don't validate private vars, but do add to config
-                # (e.g., so that our hidden _raw_split_size is included)
-                if not key.startswith("_"):
-                    user_names.append(key)
-                val = getattr(custom_cfg, key)
-                logger.debug("Overwriting: %s -> %s" % (key, val))
-                setattr(config, key, val)
     config.config_path = config_path
 
     # 3. Overrides via command-line switches

mne_bids_pipeline/_logging.py

@@ -154,3 +154,7 @@ def gen_log_kwargs(
 
 def _linkfile(uri):
     return f"[link=file://{uri}]{uri}[/link]"
+
+
+def _is_testing() -> bool:
+    return os.getenv("_MNE_BIDS_STUDY_TESTING", "") == "true"