From 58ecf2c64e73fc8f4b133c8622e95f8a045281ee Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 14:29:07 +0100 Subject: [PATCH 1/9] add aeon-neuro --- docs/installation.md | 11 ++++ pyproject.toml | 4 ++ tsml_eval/experiments/_get_classifier.py | 63 +++++++++++++++++++ .../experiments/tests/test_classification.py | 55 ++++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index 9271cf777..9758e5c0b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -25,6 +25,13 @@ required, or install all of them using the `all_extras` extra dependency set: pip install tsml-eval[all_extras] ``` +To run EEG classifiers from `aeon-neuro` 0.3.0 or later, install the `neuro` +extra: + +```console +pip install tsml-eval[neuro] +``` + ```{note} If this results in a "no matches found" error, it may be due to how your shell handles special characters. Try surrounding the dependency portion with quotes i.e. @@ -32,6 +39,10 @@ pip install tsml-eval[all_extras] pip install tsml-eval"[all_extras]" ``` +Use an aeon-neuro classifier by its class name, for example +`get_classifier_by_name("EEGNetClassifier")`. Newly exported aeon-neuro +classifiers can also be selected as `neuro:ClassName`. + All extra dependency sets can be found in the [pyproject.toml](https://github.com/time-series-machine-learning/tsml-eval/blob/main/pyproject.toml) file `[project.optional-dependencies]` options. diff --git a/pyproject.toml b/pyproject.toml index 654eba69f..b01c7e8e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ [project.optional-dependencies] all_extras = [ "aeon[all_extras]", + "aeon-neuro>=0.3.0", "tsml[all_extras]", "xgboost", ] @@ -60,6 +61,9 @@ unstable_extras = [ deep_learning = [ "aeon[dl]", ] +neuro = [ + "aeon-neuro>=0.3.0", +] dev = [ "pre-commit", diff --git a/tsml_eval/experiments/_get_classifier.py b/tsml_eval/experiments/_get_classifier.py index 9d1e00d4d..8736f6580 100644 --- a/tsml_eval/experiments/_get_classifier.py +++ b/tsml_eval/experiments/_get_classifier.py @@ -30,6 +30,13 @@ ["individualliteclassifier", "individuallite"], ["disjointcnnclassifier", "disjointcnn"], ] +neuro_classifiers = [ + ["eegnetclassifier", "eegnet"], + ["deepconvnetclassifier", "deepconvnet"], + ["riemannianmdmclassifier", "riemannianmdm"], + ["riemannianknnclassifier", "riemannianknn"], + ["cspsvmclassifier", "cspsvm"], +] dictionary_based_classifiers = [ ["bossensemble", "boss"], "individualboss", @@ -189,6 +196,8 @@ def get_classifier_by_name( return _set_classifier_interval_based( c, random_state, n_jobs, fit_contract, checkpoint, kwargs ) + elif str_in_nested_list(neuro_classifiers, c): + return _set_classifier_neuro(c, random_state, n_jobs, kwargs) elif str_in_nested_list(other_classifiers, c): return _set_classifier_other( c, random_state, n_jobs, fit_contract, checkpoint, kwargs @@ -201,6 +210,10 @@ def get_classifier_by_name( return _set_classifier_vector( c, random_state, n_jobs, fit_contract, checkpoint, kwargs ) + elif c.startswith("neuro:"): + return _set_classifier_neuro_by_class_name( + c.removeprefix("neuro:"), random_state, n_jobs, kwargs + ) else: raise ValueError(f"UNKNOWN CLASSIFIER: {c} in get_classifier_by_name") @@ -346,6 +359,56 @@ def _set_classifier_deep_learning( raise ValueError(f"UNKNOWN CLASSIFIER: {c} in get_classifier_by_name") +def _set_classifier_neuro(c, random_state, n_jobs, kwargs): + if c in ("eegnetclassifier", "eegnet"): + from aeon_neuro.classification.deep_learning import EEGNetClassifier + + return EEGNetClassifier(random_state=random_state, **kwargs) + elif c in ("deepconvnetclassifier", "deepconvnet"): + from aeon_neuro.classification.deep_learning import DeepConvNetClassifier + + return DeepConvNetClassifier(random_state=random_state, **kwargs) + elif c in ("riemannianmdmclassifier", "riemannianmdm"): + from aeon_neuro.classification.distance_based import RiemannianMDMClassifier + + return RiemannianMDMClassifier(n_jobs=n_jobs, **kwargs) + elif c in ("riemannianknnclassifier", "riemannianknn"): + from aeon_neuro.classification.distance_based import RiemannianKNNClassifier + + return RiemannianKNNClassifier(n_jobs=n_jobs, **kwargs) + elif c in ("cspsvmclassifier", "cspsvm"): + from aeon_neuro.classification.feature_based import CSPSVMClassifier + + return CSPSVMClassifier(random_state=random_state, **kwargs) + else: + raise ValueError(f"UNKNOWN CLASSIFIER: {c} in get_classifier_by_name") + + +def _set_classifier_neuro_by_class_name(c, random_state, n_jobs, kwargs): + """Create an aeon-neuro classifier exported by its classification package.""" + from inspect import isclass, signature + + from aeon.classification.base import BaseClassifier + from aeon_neuro import classification + + for classifier in vars(classification).values(): + if ( + isclass(classifier) + and classifier.__name__.casefold() == c + and classifier.__module__.startswith("aeon_neuro.") + and issubclass(classifier, BaseClassifier) + ): + parameters = signature(classifier).parameters + init_kwargs = kwargs.copy() + if "random_state" in parameters: + init_kwargs.setdefault("random_state", random_state) + if "n_jobs" in parameters: + init_kwargs.setdefault("n_jobs", n_jobs) + return classifier(**init_kwargs) + + raise ValueError(f"UNKNOWN CLASSIFIER: neuro:{c} in get_classifier_by_name") + + def _set_classifier_dictionary_based( c, random_state, n_jobs, fit_contract, checkpoint, kwargs ): diff --git a/tsml_eval/experiments/tests/test_classification.py b/tsml_eval/experiments/tests/test_classification.py index 7a69451f1..48354f22d 100644 --- a/tsml_eval/experiments/tests/test_classification.py +++ b/tsml_eval/experiments/tests/test_classification.py @@ -215,6 +215,61 @@ def test_get_classifier_by_name_invalid(): get_classifier_by_name("invalid") +@pytest.mark.parametrize( + "name, expected", + [ + ("EEGNet", "EEGNetClassifier"), + ("DeepConvNet", "DeepConvNetClassifier"), + ("RiemannianMDM", "RiemannianMDMClassifier"), + ("RiemannianKNN", "RiemannianKNNClassifier"), + ("CSPSVM", "CSPSVMClassifier"), + ], +) +def test_get_aeon_neuro_classifier(name, expected): + """Test native aeon-neuro classifiers are available by name and alias.""" + pytest.importorskip("aeon_neuro") + for classifier_name in (name, expected, f"neuro:{expected}"): + try: + classifier = get_classifier_by_name(classifier_name) + except ModuleNotFoundError as exc: + pytest.skip(str(exc)) + assert type(classifier).__name__ == expected + + +def test_get_aeon_neuro_classifier_invalid(): + """Test that the prefixed lookup only accepts exported neuro classifiers.""" + pytest.importorskip("aeon_neuro") + with pytest.raises(ValueError, match="UNKNOWN CLASSIFIER"): + get_classifier_by_name("neuro:InvalidClassifier") + + +def test_get_future_aeon_neuro_classifier(monkeypatch): + """Test discovery of a newly exported classifier and shared parameters.""" + classification = pytest.importorskip("aeon_neuro.classification") + from aeon.classification.base import BaseClassifier + + class FutureClassifier(BaseClassifier): + __module__ = "aeon_neuro.classification.future" + + def __init__(self, random_state=None, n_jobs=1): + self.random_state = random_state + self.n_jobs = n_jobs + super().__init__() + + def _fit(self, X, y): + return self + + def _predict(self, X): + return [0] * len(X) + + monkeypatch.setattr(classification, "FutureClassifier", FutureClassifier, raising=False) + classifier = get_classifier_by_name( + "neuro:FutureClassifier", random_state=42, n_jobs=2 + ) + assert isinstance(classifier, FutureClassifier) + assert (classifier.random_state, classifier.n_jobs) == (42, 2) + + def test_aeon_classifiers_available(): """Test all aeon classifiers are available.""" excluded = [ From ab6cb3326135cfab8be7c8de922cf120e6142d3f Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 14:42:11 +0100 Subject: [PATCH 2/9] add guide --- _tsml_research_resources/soton/iridis/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/_tsml_research_resources/soton/iridis/README.md b/_tsml_research_resources/soton/iridis/README.md index db3ecadcf..3d0aacb09 100644 --- a/_tsml_research_resources/soton/iridis/README.md +++ b/_tsml_research_resources/soton/iridis/README.md @@ -1,3 +1,11 @@ +eeg_classification_students.md: + Student guide for downloading aeon-neuro EEG problems and running aeon and + aeon-neuro classification experiments on Iridis 6. + +download_eeg_classification.py: + Downloads the available EEG classification problems with aeon-neuro and writes + a dataset list for the submission scripts. + serial_scripts: Simple(er) scripts which submit each experiment as a separate job and requests 1 core. This is fine for small numbers of experiments. From 53c608a1663f21b91c810cd1abc5a234bcc948be Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 14:42:15 +0100 Subject: [PATCH 3/9] add guide --- .../iridis/download_eeg_classification.py | 78 +++++++ .../iridis/eeg_classification_students.md | 211 ++++++++++++++++++ .../tests/test_download_eeg_classification.py | 57 +++++ 3 files changed, 346 insertions(+) create mode 100644 _tsml_research_resources/soton/iridis/download_eeg_classification.py create mode 100644 _tsml_research_resources/soton/iridis/eeg_classification_students.md create mode 100644 tsml_eval/experiments/tests/test_download_eeg_classification.py diff --git a/_tsml_research_resources/soton/iridis/download_eeg_classification.py b/_tsml_research_resources/soton/iridis/download_eeg_classification.py new file mode 100644 index 000000000..a7a8b67e7 --- /dev/null +++ b/_tsml_research_resources/soton/iridis/download_eeg_classification.py @@ -0,0 +1,78 @@ +"""Download aeon-neuro EEG classification problems for tsml-eval experiments. + +Run this on an Iridis node with internet access. The output dataset list can be passed +to the Iridis classification submission scripts. +""" + +import argparse +from pathlib import Path + + +def main(): + """Download published problems and write a list of usable datasets.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-dir", type=Path, required=True) + parser.add_argument("--dataset-list", type=Path, required=True) + parser.add_argument( + "--only", nargs="+", metavar="NAME", help="Download selected datasets first" + ) + args = parser.parse_args() + + from aeon_neuro.datasets.classification_datasets import dataset_map, dataset_names + + names = args.only if args.only else dataset_names + unknown = sorted(set(names) - set(dataset_names)) + if unknown: + parser.error(f"Unknown aeon-neuro datasets: {', '.join(unknown)}") + + args.data_dir.mkdir(parents=True, exist_ok=True) + args.dataset_list.parent.mkdir(parents=True, exist_ok=True) + ready = [] + unavailable = [] + failed = [] + + for name in names: + problem_dir = args.data_dir / name + train = problem_dir / f"{name}_TRAIN.ts" + test = problem_dir / f"{name}_TEST.ts" + if train.is_file() and test.is_file(): + print(f"READY {name} (already downloaded)", flush=True) + ready.append(name) + continue + + if dataset_map.get(name) == 49: + print(f"UNAVAILABLE {name} (no download record yet)", flush=True) + unavailable.append(name) + continue + + try: + # The loader downloads both splits; loading only TRAIN limits memory use. + from aeon_neuro.datasets import load_eeg_classification + + load_eeg_classification(name, split="train", extract_path=str(args.data_dir)) + if not train.is_file() or not test.is_file(): + raise FileNotFoundError(f"Missing TRAIN or TEST file in {problem_dir}") + except Exception as exc: + print(f"FAILED {name}: {exc}", flush=True) + failed.append(name) + continue + + print(f"READY {name}", flush=True) + ready.append(name) + + args.dataset_list.write_text("".join(f"{name}\n" for name in ready)) + print( + f"Ready: {len(ready)}; unavailable: {len(unavailable)}; failed: " + f"{len(failed)}. Dataset list: {args.dataset_list}", + flush=True, + ) + if unavailable: + print(f"Unavailable: {', '.join(unavailable)}", flush=True) + if failed: + print(f"Failed: {', '.join(failed)}", flush=True) + if unavailable or failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/_tsml_research_resources/soton/iridis/eeg_classification_students.md b/_tsml_research_resources/soton/iridis/eeg_classification_students.md new file mode 100644 index 000000000..1edde0f89 --- /dev/null +++ b/_tsml_research_resources/soton/iridis/eeg_classification_students.md @@ -0,0 +1,211 @@ +# EEG classification experiments on Iridis 6 + +This guide takes you from a fresh Iridis account to tsml-eval prediction files for +classifiers from aeon and aeon-neuro. Commands below are for Bash on Iridis 6. The +[general Iridis Python guide](iridis_python.md) covers cluster access, storage, Conda, +and the supplied submission scripts in more detail. Check the [HPC Community Wiki](https://sotonac.sharepoint.com/teams/HPCCommunityWiki) +for current queue rules and account instructions. + +## Before you start + +You need an Iridis account, access to the Southampton network or VPN, and an +aeon-neuro release containing the EEG dataset downloader and classifiers. This guide +expects aeon-neuro **0.3.0 or later** and a tsml-eval checkout containing its `neuro` +extra. If the tsml-eval changes have not been merged into `main`, use the branch +provided by your supervisor. The code below uses Python 3.13 because the aeon-neuro +checkout inspected for this guide does not declare Python 3.14 support. + +The EEG dataset downloader depends on published download records. At the time this +guide was written, the local aeon-neuro dataset map listed 31 problems but marked +every record `49` (unavailable). The download step will report these names and write +an empty dataset list until the records are provided. **Do not submit experiments +against an empty list.** Ask your supervisor for the updated aeon-neuro release or +staged `.ts` files if this happens. + +## 1. Connect and install the software + +Connect to `iridis6.soton.ac.uk` over SSH. Run these commands on the login node, +where Git and package downloads are available: + +```bash +module load conda/python3 +conda create -n tsml-neuro python=3.13 +conda activate tsml-neuro +git clone https://github.com/time-series-machine-learning/tsml-eval.git "$HOME/tsml-eval" +cd "$HOME/tsml-eval" +# If the aeon-neuro changes are not yet on main, checkout the supplied branch here. +python -m pip install --editable '.[neuro]' +python -c 'import aeon, aeon_neuro, tsml_eval; print(aeon.__version__, aeon_neuro.__version__, tsml_eval.__version__)' +``` + +If you intend to run aeon's TensorFlow classifiers such as +`InceptionTimeClassifier`, also install the `deep_learning` extra in the same +environment (or use a separate GPU environment): + +```bash +python -m pip install --editable '.[neuro,deep_learning]' +``` + +The editable install makes the Python files in your clone the ones that run. Keep the +clone at a known commit while collecting results; record `git rev-parse HEAD` and +`python -m pip freeze` alongside the results. + +## 2. Stage the EEG problems once + +Put datasets and results on your scratch storage, not inside the Git checkout. The +supplied Python script calls `load_eeg_classification` for each listed EEG problem, +checks that both `.ts` splits exist, and writes only usable dataset names to a text +file. Each line in that file is consumed by the Iridis submission scripts. + +```bash +EEG_WORK="/iridisfs/scratch/$USER/eeg" +mkdir -p "$EEG_WORK/data" "$EEG_WORK/results" "$EEG_WORK/output" +python "$HOME/tsml-eval/_tsml_research_resources/soton/iridis/download_eeg_classification.py" \ + --data-dir "$EEG_WORK/data" \ + --dataset-list "$EEG_WORK/datasets.txt" +cat "$EEG_WORK/datasets.txt" +``` + +Run this download step on a node that has internet access. The [Iridis Python +guide](iridis_python.md) notes that compute nodes may not have internet access. The +script processes one problem at a time but does load its training split into memory; +for very large problems, ask the cluster team or your supervisor for a suitable +transfer method rather than running a heavy download on the login node. You can +check a single problem first with `--only SelfRegulationSCP1`. + +The resulting layout must be: + +```text +eeg/data/SelfRegulationSCP1/SelfRegulationSCP1_TRAIN.ts +eeg/data/SelfRegulationSCP1/SelfRegulationSCP1_TEST.ts +eeg/datasets.txt +``` + +If some names are unavailable or a download fails, the script exits with an error +and prints a summary. It still writes the successfully staged names to +`datasets.txt`, so you can rerun it after the source problem is fixed. Existing +complete train/test pairs are reused. Inspect the list before submitting jobs. + +## 3. Run one smoke test + +First run one small experiment in an interactive compute session (`sinteractive` +on Iridis). Do this after the dataset files have been staged; compute nodes may not +be able to download them. In the new session, reactivate the environment: + +```bash +module load conda/python3 +conda activate tsml-neuro +EEG_WORK="/iridisfs/scratch/$USER/eeg" +python -u "$HOME/tsml-eval/tsml_eval/experiments/classification_experiments.py" \ + "$EEG_WORK/data" "$EEG_WORK/results" \ + ROCKET SelfRegulationSCP1 0 +``` + +Replace `SelfRegulationSCP1` with a name actually present in your +`datasets.txt`. `ROCKET` is an aeon classifier. For an aeon-neuro CPU classifier, +replace it with `RiemannianMDMClassifier` or `CSPSVMClassifier`. The classifier +names and aliases implemented in this checkout are in +[`tsml_eval/experiments/_get_classifier.py`](../../../tsml_eval/experiments/_get_classifier.py). +New aeon-neuro classes exported by `aeon_neuro.classification` can be requested as +`neuro:ClassName`. + +The final positional argument `0` uses the supplied train/test split and is also +the result-file suffix. The file should appear at: + +```text +eeg/results/ROCKET/Predictions/SelfRegulationSCP1/testResample0.csv +``` + +The script skips a completed result by default. Add `-ow` only when you intend +to overwrite it. Add `-tr` if you also need a training results file; that can be +substantially slower because it may perform cross-validation. For nonzero resample +IDs, tsml-eval stratifies a new train/test split and writes, for example, +`testResample1.csv`. + +## 4. Submit CPU experiments across the dataset list + +The existing [serial classification submission script](serial_scripts/classification_experiments.sh) +submits Slurm array jobs for each dataset and classifier. Make a personal copy so +your paths and scheduler choices do not affect other students: + +```bash +cp "$HOME/tsml-eval/_tsml_research_resources/soton/iridis/serial_scripts/classification_experiments.sh" \ + "$HOME/eeg_classification_cpu.sh" +``` + +Edit the configuration block near the top of your copy. For a first pass, use one +fold (the original split), a modest submission limit, and your own paths: + +```bash +max_folds=1 +start_fold=1 +max_num_submitted=10 +queue="batch" +username="$USER" +local_path="$HOME/" +data_dir="/iridisfs/scratch/$USER/eeg/data/" +datasets="/iridisfs/scratch/$USER/eeg/datasets.txt" +results_dir="/iridisfs/scratch/$USER/eeg/results/" +out_dir="/iridisfs/scratch/$USER/eeg/output/" +script_file_path="$HOME/tsml-eval/tsml_eval/experiments/classification_experiments.py" +env_name="tsml-neuro" +classifiers_to_run="ROCKET RiemannianMDMClassifier" +generate_train_files="false" +predefined_folds="false" +normalise_data="false" +``` + +Keep the trailing `/` on directory settings; the script concatenates names onto +these paths. Check `max_memory` and `max_time` in the same block. Increase them +only if jobs show they need more resources. The script requests one CPU per task, +and the Python experiment script limits common thread pools to one. + +After checking that `datasets.txt` contains the intended names, leave the Conda +environment and run the **Bash** submission script on the login node: + +```bash +conda deactivate +cd "/iridisfs/scratch/$USER/eeg" +bash "$HOME/eeg_classification_cpu.sh" +squeue -u "$USER" +``` + +The script uses Bash-specific syntax, so run it with `bash`, not `sh`. It skips +result files that already exist and writes Slurm logs under `eeg/output/`. If +you need more resamples later, increase `max_folds`; fold 1 maps to result +`Resample0`, fold 2 to `Resample1`, and so on. + +## 5. GPU classifiers + +For aeon-neuro deep learning models such as `EEGNetClassifier` and +`DeepConvNetClassifier`, follow the [GPU environment instructions](iridis_python.md#51-tsml-eval-gpu) +to build an Apptainer sandbox with the required packages. Install this tsml-eval +checkout and its `neuro` extra **inside the sandbox**. Copy the +[GPU classification submission script](gpu_scripts/gpu_classification_experiments.sh) +and set its `data_dir`, `datasets`, `results_dir`, `out_dir`, +`script_file_path`, `container_path`, `username`, `classifiers_to_run`, and +fold and resource settings for your account. Set `classifiers_to_run` initially +to one model and one dataset. That script requests one GPU per array task on the +`gpu` partition and calls the same tsml-eval Python experiment file via +`apptainer exec --nv`. + +Confirm in a GPU job that the container can read the cloned source and scratch +data paths, imports `aeon_neuro`, and sees the GPU before submitting the full +dataset list. Do not assume that packages installed in your host Conda environment +are available inside the container. + +## 6. Check and keep the results + +Each successful test experiment writes a CSV under +`//Predictions//testResample.csv`. +If `-tr` was requested, there is also a `trainResample.csv`. The file begins +with experiment metadata and then stores predictions and probabilities. The +[results format notebook](../../../examples/results_format.ipynb) explains the +format and the [evaluation notebook](../../../examples/evaluation.ipynb) shows +how to compare classifiers. + +For a failed job, inspect the corresponding `.err` and `.out` files in +`eeg/output///`. If the submission script prints that a job +was skipped, check whether the expected result file already exists. Keep the +dataset list, the tsml-eval Git commit, package versions, and job settings with +the CSV files so experiments can be reproduced. diff --git a/tsml_eval/experiments/tests/test_download_eeg_classification.py b/tsml_eval/experiments/tests/test_download_eeg_classification.py new file mode 100644 index 000000000..9dc0c390e --- /dev/null +++ b/tsml_eval/experiments/tests/test_download_eeg_classification.py @@ -0,0 +1,57 @@ +"""Tests for the Iridis EEG dataset download helper.""" + +import runpy +import sys +from pathlib import Path +from types import ModuleType + +import pytest + + +def test_download_eeg_classification_writes_only_ready_datasets(monkeypatch, tmp_path): + """Include staged and downloaded datasets, and report unavailable records.""" + data_dir = tmp_path / "data" + ready_dir = data_dir / "Ready" + ready_dir.mkdir(parents=True) + (ready_dir / "Ready_TRAIN.ts").write_text("train") + (ready_dir / "Ready_TEST.ts").write_text("test") + dataset_list = tmp_path / "datasets.txt" + + datasets = ModuleType("aeon_neuro.datasets") + datasets.__path__ = [] + collection = ModuleType("aeon_neuro.datasets.classification_datasets") + collection.dataset_names = ["Ready", "Download", "Unavailable"] + collection.dataset_map = {"Ready": 1, "Download": 2, "Unavailable": 49} + + def load_eeg_classification(name, split, extract_path): + assert (name, split) == ("Download", "train") + problem_dir = Path(extract_path) / name + problem_dir.mkdir() + (problem_dir / f"{name}_TRAIN.ts").write_text("train") + (problem_dir / f"{name}_TEST.ts").write_text("test") + + datasets.load_eeg_classification = load_eeg_classification + monkeypatch.setitem(sys.modules, "aeon_neuro.datasets", datasets) + monkeypatch.setitem( + sys.modules, "aeon_neuro.datasets.classification_datasets", collection + ) + monkeypatch.setattr( + sys, + "argv", + [ + "download_eeg_classification.py", + "--data-dir", + str(data_dir), + "--dataset-list", + str(dataset_list), + ], + ) + + script = ( + Path(__file__).parents[3] + / "_tsml_research_resources/soton/iridis/download_eeg_classification.py" + ) + with pytest.raises(SystemExit, match="1"): + runpy.run_path(str(script), run_name="__main__") + + assert dataset_list.read_text().splitlines() == ["Ready", "Download"] From 628a6886ba6e4398437f5395b6ab2389df800071 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 14:49:41 +0100 Subject: [PATCH 4/9] Download available EEG ID sidecars --- .../iridis/download_eeg_classification.py | 89 ++++++++++++++----- .../iridis/eeg_classification_students.md | 12 ++- .../tests/test_download_eeg_classification.py | 13 +++ 3 files changed, 92 insertions(+), 22 deletions(-) diff --git a/_tsml_research_resources/soton/iridis/download_eeg_classification.py b/_tsml_research_resources/soton/iridis/download_eeg_classification.py index a7a8b67e7..4d5a371e1 100644 --- a/_tsml_research_resources/soton/iridis/download_eeg_classification.py +++ b/_tsml_research_resources/soton/iridis/download_eeg_classification.py @@ -5,7 +5,40 @@ """ import argparse +import shutil from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import urlopen + + +def download_id_files(name, record_id, problem_dir): + """Fetch subject/case ID sidecars when they are present in the Zenodo record.""" + for split in ("TRAIN", "TEST"): + filename = f"{name}_id_{split}.txt" + target = problem_dir / filename + if target.is_file(): + continue + + url = ( + f"https://zenodo.org/records/{record_id}/files/" + f"{quote(filename)}?download=1" + ) + partial = problem_dir / f"{filename}.part" + try: + with urlopen(url, timeout=60) as response, partial.open("wb") as output: + shutil.copyfileobj(response, output) + partial.replace(target) + except HTTPError as exc: + if exc.code != 404: + raise + finally: + partial.unlink(missing_ok=True) + + return all( + (problem_dir / f"{name}_id_{split}.txt").is_file() + for split in ("TRAIN", "TEST") + ) def main(): @@ -30,32 +63,43 @@ def main(): ready = [] unavailable = [] failed = [] + id_complete = [] + id_failed = [] for name in names: problem_dir = args.data_dir / name train = problem_dir / f"{name}_TRAIN.ts" test = problem_dir / f"{name}_TEST.ts" - if train.is_file() and test.is_file(): - print(f"READY {name} (already downloaded)", flush=True) - ready.append(name) - continue + record_id = dataset_map.get(name) + if not (train.is_file() and test.is_file()): + if record_id in (None, 49): + print(f"UNAVAILABLE {name} (no download record yet)", flush=True) + unavailable.append(name) + continue - if dataset_map.get(name) == 49: - print(f"UNAVAILABLE {name} (no download record yet)", flush=True) - unavailable.append(name) - continue + try: + # The loader downloads both splits; load only TRAIN to limit memory. + from aeon_neuro.datasets import load_eeg_classification - try: - # The loader downloads both splits; loading only TRAIN limits memory use. - from aeon_neuro.datasets import load_eeg_classification - - load_eeg_classification(name, split="train", extract_path=str(args.data_dir)) - if not train.is_file() or not test.is_file(): - raise FileNotFoundError(f"Missing TRAIN or TEST file in {problem_dir}") - except Exception as exc: - print(f"FAILED {name}: {exc}", flush=True) - failed.append(name) - continue + load_eeg_classification( + name, split="train", extract_path=str(args.data_dir) + ) + if not train.is_file() or not test.is_file(): + raise FileNotFoundError( + f"Missing TRAIN or TEST file in {problem_dir}" + ) + except Exception as exc: + print(f"FAILED {name}: {exc}", flush=True) + failed.append(name) + continue + + if record_id not in (None, 49): + try: + if download_id_files(name, record_id, problem_dir): + id_complete.append(name) + except Exception as exc: + print(f"ID DOWNLOAD FAILED {name}: {exc}", flush=True) + id_failed.append(name) print(f"READY {name}", flush=True) ready.append(name) @@ -63,14 +107,17 @@ def main(): args.dataset_list.write_text("".join(f"{name}\n" for name in ready)) print( f"Ready: {len(ready)}; unavailable: {len(unavailable)}; failed: " - f"{len(failed)}. Dataset list: {args.dataset_list}", + f"{len(failed)}; paired ID files: {len(id_complete)}. " + f"Dataset list: {args.dataset_list}", flush=True, ) if unavailable: print(f"Unavailable: {', '.join(unavailable)}", flush=True) if failed: print(f"Failed: {', '.join(failed)}", flush=True) - if unavailable or failed: + if id_failed: + print(f"ID download failed: {', '.join(id_failed)}", flush=True) + if unavailable or failed or id_failed: raise SystemExit(1) diff --git a/_tsml_research_resources/soton/iridis/eeg_classification_students.md b/_tsml_research_resources/soton/iridis/eeg_classification_students.md index 1edde0f89..78b81d939 100644 --- a/_tsml_research_resources/soton/iridis/eeg_classification_students.md +++ b/_tsml_research_resources/soton/iridis/eeg_classification_students.md @@ -55,7 +55,10 @@ clone at a known commit while collecting results; record `git rev-parse HEAD` an Put datasets and results on your scratch storage, not inside the Git checkout. The supplied Python script calls `load_eeg_classification` for each listed EEG problem, checks that both `.ts` splits exist, and writes only usable dataset names to a text -file. Each line in that file is consumed by the Iridis submission scripts. +file. It also retrieves the paired `_id_TRAIN.txt` and `_id_TEST.txt` files when +they are present in the dataset's Zenodo record. These contain case or subject IDs +in the archive. Each line in the dataset list is consumed by the Iridis submission +scripts. ```bash EEG_WORK="/iridisfs/scratch/$USER/eeg" @@ -81,6 +84,13 @@ eeg/data/SelfRegulationSCP1/SelfRegulationSCP1_TEST.ts eeg/datasets.txt ``` +Some problems also have `_id_TRAIN.txt` and `_id_TEST.txt` next to +their `.ts` files. The downloader reports how many complete ID pairs it found. +The `.ts` format preserves channel order and sometimes describes electrode names +in header comments, but the archive does not consistently provide a separate +channel-name file. Do not treat a channel index as a named electrode without +checking that dataset's header or original documentation. + If some names are unavailable or a download fails, the script exits with an error and prints a summary. It still writes the successfully staged names to `datasets.txt`, so you can rerun it after the source problem is fixed. Existing diff --git a/tsml_eval/experiments/tests/test_download_eeg_classification.py b/tsml_eval/experiments/tests/test_download_eeg_classification.py index 9dc0c390e..1a719fbdd 100644 --- a/tsml_eval/experiments/tests/test_download_eeg_classification.py +++ b/tsml_eval/experiments/tests/test_download_eeg_classification.py @@ -2,8 +2,10 @@ import runpy import sys +from io import BytesIO from pathlib import Path from types import ModuleType +from urllib.error import HTTPError import pytest @@ -31,6 +33,14 @@ def load_eeg_classification(name, split, extract_path): (problem_dir / f"{name}_TEST.ts").write_text("test") datasets.load_eeg_classification = load_eeg_classification + + def urlopen(url, timeout): + assert timeout == 60 + if "/records/1/" in url: + return BytesIO(b"subject-id\n") + raise HTTPError(url, 404, "No ID sidecar", None, None) + + monkeypatch.setattr("urllib.request.urlopen", urlopen) monkeypatch.setitem(sys.modules, "aeon_neuro.datasets", datasets) monkeypatch.setitem( sys.modules, "aeon_neuro.datasets.classification_datasets", collection @@ -55,3 +65,6 @@ def load_eeg_classification(name, split, extract_path): runpy.run_path(str(script), run_name="__main__") assert dataset_list.read_text().splitlines() == ["Ready", "Download"] + assert (ready_dir / "Ready_id_TRAIN.txt").read_text() == "subject-id\n" + assert (ready_dir / "Ready_id_TEST.txt").read_text() == "subject-id\n" + assert not (data_dir / "Download" / "Download_id_TRAIN.txt").exists() From 79425ce4ed8d0c2c8d8d5dde6a7e41dcbd17a829 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 14:59:41 +0100 Subject: [PATCH 5/9] Use aeon Zenodo download for EEG ID sidecars --- .../iridis/download_eeg_classification.py | 50 +++---------------- .../iridis/eeg_classification_students.md | 23 +++++---- .../tests/test_download_eeg_classification.py | 12 +---- 3 files changed, 21 insertions(+), 64 deletions(-) diff --git a/_tsml_research_resources/soton/iridis/download_eeg_classification.py b/_tsml_research_resources/soton/iridis/download_eeg_classification.py index 4d5a371e1..e6524f378 100644 --- a/_tsml_research_resources/soton/iridis/download_eeg_classification.py +++ b/_tsml_research_resources/soton/iridis/download_eeg_classification.py @@ -5,40 +5,7 @@ """ import argparse -import shutil from pathlib import Path -from urllib.error import HTTPError -from urllib.parse import quote -from urllib.request import urlopen - - -def download_id_files(name, record_id, problem_dir): - """Fetch subject/case ID sidecars when they are present in the Zenodo record.""" - for split in ("TRAIN", "TEST"): - filename = f"{name}_id_{split}.txt" - target = problem_dir / filename - if target.is_file(): - continue - - url = ( - f"https://zenodo.org/records/{record_id}/files/" - f"{quote(filename)}?download=1" - ) - partial = problem_dir / f"{filename}.part" - try: - with urlopen(url, timeout=60) as response, partial.open("wb") as output: - shutil.copyfileobj(response, output) - partial.replace(target) - except HTTPError as exc: - if exc.code != 404: - raise - finally: - partial.unlink(missing_ok=True) - - return all( - (problem_dir / f"{name}_id_{split}.txt").is_file() - for split in ("TRAIN", "TEST") - ) def main(): @@ -64,7 +31,6 @@ def main(): unavailable = [] failed = [] id_complete = [] - id_failed = [] for name in names: problem_dir = args.data_dir / name @@ -93,13 +59,11 @@ def main(): failed.append(name) continue - if record_id not in (None, 49): - try: - if download_id_files(name, record_id, problem_dir): - id_complete.append(name) - except Exception as exc: - print(f"ID DOWNLOAD FAILED {name}: {exc}", flush=True) - id_failed.append(name) + if all( + (problem_dir / f"{name}_id_{split}.txt").is_file() + for split in ("TRAIN", "TEST") + ): + id_complete.append(name) print(f"READY {name}", flush=True) ready.append(name) @@ -115,9 +79,7 @@ def main(): print(f"Unavailable: {', '.join(unavailable)}", flush=True) if failed: print(f"Failed: {', '.join(failed)}", flush=True) - if id_failed: - print(f"ID download failed: {', '.join(id_failed)}", flush=True) - if unavailable or failed or id_failed: + if unavailable or failed: raise SystemExit(1) diff --git a/_tsml_research_resources/soton/iridis/eeg_classification_students.md b/_tsml_research_resources/soton/iridis/eeg_classification_students.md index 78b81d939..d439c1b1c 100644 --- a/_tsml_research_resources/soton/iridis/eeg_classification_students.md +++ b/_tsml_research_resources/soton/iridis/eeg_classification_students.md @@ -15,12 +15,11 @@ extra. If the tsml-eval changes have not been merged into `main`, use the branch provided by your supervisor. The code below uses Python 3.13 because the aeon-neuro checkout inspected for this guide does not declare Python 3.14 support. -The EEG dataset downloader depends on published download records. At the time this -guide was written, the local aeon-neuro dataset map listed 31 problems but marked -every record `49` (unavailable). The download step will report these names and write -an empty dataset list until the records are provided. **Do not submit experiments -against an empty list.** Ask your supervisor for the updated aeon-neuro release or -staged `.ts` files if this happens. +The EEG dataset downloader depends on published download records. Some entries in +aeon-neuro's dataset map may still be marked `49` (unavailable). The download step +reports these names. **Do not submit experiments against an empty dataset list.** +Ask your supervisor for an updated aeon-neuro release or staged `.ts` files if this +happens. ## 1. Connect and install the software @@ -55,10 +54,11 @@ clone at a known commit while collecting results; record `git rev-parse HEAD` an Put datasets and results on your scratch storage, not inside the Git checkout. The supplied Python script calls `load_eeg_classification` for each listed EEG problem, checks that both `.ts` splits exist, and writes only usable dataset names to a text -file. It also retrieves the paired `_id_TRAIN.txt` and `_id_TEST.txt` files when -they are present in the dataset's Zenodo record. These contain case or subject IDs -in the archive. Each line in the dataset list is consumed by the Iridis submission -scripts. +file. At the aeon-neuro repository's main HEAD, that loader calls aeon's +`load_classification`, which downloads every file attached to the Zenodo record, +including paired `_id_TRAIN.txt` and `_id_TEST.txt` files when published. These +contain case or subject IDs in the archive. Each line in the dataset list is +consumed by the Iridis submission scripts. ```bash EEG_WORK="/iridisfs/scratch/$USER/eeg" @@ -86,6 +86,9 @@ eeg/datasets.txt Some problems also have `_id_TRAIN.txt` and `_id_TEST.txt` next to their `.ts` files. The downloader reports how many complete ID pairs it found. +If the `.ts` files were staged before the current aeon download behavior, the loader +reuses them and does not fetch missing sidecars; stage those from their Zenodo record +if you need case IDs. The `.ts` format preserves channel order and sometimes describes electrode names in header comments, but the archive does not consistently provide a separate channel-name file. Do not treat a channel index as a named electrode without diff --git a/tsml_eval/experiments/tests/test_download_eeg_classification.py b/tsml_eval/experiments/tests/test_download_eeg_classification.py index 1a719fbdd..9b98cdf43 100644 --- a/tsml_eval/experiments/tests/test_download_eeg_classification.py +++ b/tsml_eval/experiments/tests/test_download_eeg_classification.py @@ -2,10 +2,8 @@ import runpy import sys -from io import BytesIO from pathlib import Path from types import ModuleType -from urllib.error import HTTPError import pytest @@ -17,6 +15,8 @@ def test_download_eeg_classification_writes_only_ready_datasets(monkeypatch, tmp ready_dir.mkdir(parents=True) (ready_dir / "Ready_TRAIN.ts").write_text("train") (ready_dir / "Ready_TEST.ts").write_text("test") + (ready_dir / "Ready_id_TRAIN.txt").write_text("subject-id\n") + (ready_dir / "Ready_id_TEST.txt").write_text("subject-id\n") dataset_list = tmp_path / "datasets.txt" datasets = ModuleType("aeon_neuro.datasets") @@ -33,14 +33,6 @@ def load_eeg_classification(name, split, extract_path): (problem_dir / f"{name}_TEST.ts").write_text("test") datasets.load_eeg_classification = load_eeg_classification - - def urlopen(url, timeout): - assert timeout == 60 - if "/records/1/" in url: - return BytesIO(b"subject-id\n") - raise HTTPError(url, 404, "No ID sidecar", None, None) - - monkeypatch.setattr("urllib.request.urlopen", urlopen) monkeypatch.setitem(sys.modules, "aeon_neuro.datasets", datasets) monkeypatch.setitem( sys.modules, "aeon_neuro.datasets.classification_datasets", collection From f0957009da7dd919d849c21dc7571731b024425f Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 16:16:21 +0100 Subject: [PATCH 6/9] Lower aeon-neuro minimum version to 0.1.0 --- .../soton/iridis/eeg_classification_students.md | 2 +- docs/installation.md | 2 +- pyproject.toml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/_tsml_research_resources/soton/iridis/eeg_classification_students.md b/_tsml_research_resources/soton/iridis/eeg_classification_students.md index d439c1b1c..73d260f84 100644 --- a/_tsml_research_resources/soton/iridis/eeg_classification_students.md +++ b/_tsml_research_resources/soton/iridis/eeg_classification_students.md @@ -10,7 +10,7 @@ for current queue rules and account instructions. You need an Iridis account, access to the Southampton network or VPN, and an aeon-neuro release containing the EEG dataset downloader and classifiers. This guide -expects aeon-neuro **0.3.0 or later** and a tsml-eval checkout containing its `neuro` +expects aeon-neuro **0.1.0 or later** and a tsml-eval checkout containing its `neuro` extra. If the tsml-eval changes have not been merged into `main`, use the branch provided by your supervisor. The code below uses Python 3.13 because the aeon-neuro checkout inspected for this guide does not declare Python 3.14 support. diff --git a/docs/installation.md b/docs/installation.md index 9758e5c0b..d0061217e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -25,7 +25,7 @@ required, or install all of them using the `all_extras` extra dependency set: pip install tsml-eval[all_extras] ``` -To run EEG classifiers from `aeon-neuro` 0.3.0 or later, install the `neuro` +To run EEG classifiers from `aeon-neuro` 0.1.0 or later, install the `neuro` extra: ```console diff --git a/pyproject.toml b/pyproject.toml index b01c7e8e4..1e6a683de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ [project.optional-dependencies] all_extras = [ "aeon[all_extras]", - "aeon-neuro>=0.3.0", + "aeon-neuro>=0.1.0", "tsml[all_extras]", "xgboost", ] @@ -62,7 +62,7 @@ deep_learning = [ "aeon[dl]", ] neuro = [ - "aeon-neuro>=0.3.0", + "aeon-neuro>=0.1.0", ] dev = [ From aff356bceaf732e738ecb195110e1b7c4a92b096 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 16:24:32 +0100 Subject: [PATCH 7/9] Use logging in Iridis EEG downloader --- .../iridis/download_eeg_classification.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/_tsml_research_resources/soton/iridis/download_eeg_classification.py b/_tsml_research_resources/soton/iridis/download_eeg_classification.py index e6524f378..f67e38c4a 100644 --- a/_tsml_research_resources/soton/iridis/download_eeg_classification.py +++ b/_tsml_research_resources/soton/iridis/download_eeg_classification.py @@ -5,11 +5,16 @@ """ import argparse +import logging +import sys from pathlib import Path +logger = logging.getLogger(__name__) + def main(): """Download published problems and write a list of usable datasets.""" + logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data-dir", type=Path, required=True) parser.add_argument("--dataset-list", type=Path, required=True) @@ -39,7 +44,7 @@ def main(): record_id = dataset_map.get(name) if not (train.is_file() and test.is_file()): if record_id in (None, 49): - print(f"UNAVAILABLE {name} (no download record yet)", flush=True) + logger.info("UNAVAILABLE %s (no download record yet)", name) unavailable.append(name) continue @@ -55,7 +60,7 @@ def main(): f"Missing TRAIN or TEST file in {problem_dir}" ) except Exception as exc: - print(f"FAILED {name}: {exc}", flush=True) + logger.info("FAILED %s: %s", name, exc) failed.append(name) continue @@ -65,20 +70,23 @@ def main(): ): id_complete.append(name) - print(f"READY {name}", flush=True) + logger.info("READY %s", name) ready.append(name) args.dataset_list.write_text("".join(f"{name}\n" for name in ready)) - print( - f"Ready: {len(ready)}; unavailable: {len(unavailable)}; failed: " - f"{len(failed)}; paired ID files: {len(id_complete)}. " - f"Dataset list: {args.dataset_list}", - flush=True, + logger.info( + "Ready: %s; unavailable: %s; failed: %s; paired ID files: %s. " + "Dataset list: %s", + len(ready), + len(unavailable), + len(failed), + len(id_complete), + args.dataset_list, ) if unavailable: - print(f"Unavailable: {', '.join(unavailable)}", flush=True) + logger.info("Unavailable: %s", ", ".join(unavailable)) if failed: - print(f"Failed: {', '.join(failed)}", flush=True) + logger.info("Failed: %s", ", ".join(failed)) if unavailable or failed: raise SystemExit(1) From de8a26f449723339db7f5f6073cbad50e936f850 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 16:26:08 +0100 Subject: [PATCH 8/9] Format aeon-neuro classifier test --- tsml_eval/experiments/tests/test_classification.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tsml_eval/experiments/tests/test_classification.py b/tsml_eval/experiments/tests/test_classification.py index 48354f22d..9cdf5c1c4 100644 --- a/tsml_eval/experiments/tests/test_classification.py +++ b/tsml_eval/experiments/tests/test_classification.py @@ -262,7 +262,9 @@ def _fit(self, X, y): def _predict(self, X): return [0] * len(X) - monkeypatch.setattr(classification, "FutureClassifier", FutureClassifier, raising=False) + monkeypatch.setattr( + classification, "FutureClassifier", FutureClassifier, raising=False + ) classifier = get_classifier_by_name( "neuro:FutureClassifier", random_state=42, n_jobs=2 ) From e8a3f837435980f5195bc4ac32e63fd59e3170fd Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Wed, 21 Oct 2026 19:34:46 +0100 Subject: [PATCH 9/9] Add CSP LDA, forest and SVM neuro classifiers --- .../iridis/eeg_classification_students.md | 4 ++- docs/installation.md | 2 +- pyproject.toml | 4 +-- tsml_eval/experiments/_get_classifier.py | 34 ++++++++++++++++--- .../experiments/tests/test_classification.py | 24 ++++++++++++- 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/_tsml_research_resources/soton/iridis/eeg_classification_students.md b/_tsml_research_resources/soton/iridis/eeg_classification_students.md index 73d260f84..aabc6467f 100644 --- a/_tsml_research_resources/soton/iridis/eeg_classification_students.md +++ b/_tsml_research_resources/soton/iridis/eeg_classification_students.md @@ -116,7 +116,9 @@ python -u "$HOME/tsml-eval/tsml_eval/experiments/classification_experiments.py" Replace `SelfRegulationSCP1` with a name actually present in your `datasets.txt`. `ROCKET` is an aeon classifier. For an aeon-neuro CPU classifier, -replace it with `RiemannianMDMClassifier` or `CSPSVMClassifier`. The classifier +replace it with `RiemannianMDMClassifier` or `CSPClassifier` (CSP with LDA). +Use `CSPRandomForest` for CSP with a random forest or `CSPSVM` for CSP with a +linear SVM. The classifier names and aliases implemented in this checkout are in [`tsml_eval/experiments/_get_classifier.py`](../../../tsml_eval/experiments/_get_classifier.py). New aeon-neuro classes exported by `aeon_neuro.classification` can be requested as diff --git a/docs/installation.md b/docs/installation.md index d0061217e..3ab46aa67 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -26,7 +26,7 @@ pip install tsml-eval[all_extras] ``` To run EEG classifiers from `aeon-neuro` 0.1.0 or later, install the `neuro` -extra: +extra with Python 3.12 or 3.13 (the Python versions supported by that release): ```console pip install tsml-eval[neuro] diff --git a/pyproject.toml b/pyproject.toml index 1e6a683de..549bf6198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ [project.optional-dependencies] all_extras = [ "aeon[all_extras]", - "aeon-neuro>=0.1.0", + "aeon-neuro>=0.1.0; python_version < '3.14'", "tsml[all_extras]", "xgboost", ] @@ -62,7 +62,7 @@ deep_learning = [ "aeon[dl]", ] neuro = [ - "aeon-neuro>=0.1.0", + "aeon-neuro>=0.1.0; python_version < '3.14'", ] dev = [ diff --git a/tsml_eval/experiments/_get_classifier.py b/tsml_eval/experiments/_get_classifier.py index 8736f6580..129921f16 100644 --- a/tsml_eval/experiments/_get_classifier.py +++ b/tsml_eval/experiments/_get_classifier.py @@ -35,6 +35,8 @@ ["deepconvnetclassifier", "deepconvnet"], ["riemannianmdmclassifier", "riemannianmdm"], ["riemannianknnclassifier", "riemannianknn"], + ["cspclassifier", "csp", "csplda"], + ["csprandomforestclassifier", "csprandomforest", "csprf"], ["cspsvmclassifier", "cspsvm"], ] dictionary_based_classifiers = [ @@ -376,10 +378,34 @@ def _set_classifier_neuro(c, random_state, n_jobs, kwargs): from aeon_neuro.classification.distance_based import RiemannianKNNClassifier return RiemannianKNNClassifier(n_jobs=n_jobs, **kwargs) - elif c in ("cspsvmclassifier", "cspsvm"): - from aeon_neuro.classification.feature_based import CSPSVMClassifier - - return CSPSVMClassifier(random_state=random_state, **kwargs) + elif c in ( + "cspclassifier", + "csp", + "csplda", + "csprandomforestclassifier", + "csprandomforest", + "csprf", + "cspsvmclassifier", + "cspsvm", + ): + from aeon_neuro.classification.feature_based import CSPClassifier + + if c in ("csprandomforestclassifier", "csprandomforest", "csprf"): + from sklearn.ensemble import RandomForestClassifier + + classifier = RandomForestClassifier( + random_state=random_state, n_jobs=n_jobs + ) + elif c in ("cspsvmclassifier", "cspsvm"): + from sklearn.svm import SVC + + classifier = SVC( + kernel="linear", probability=True, random_state=random_state + ) + else: + classifier = None + kwargs.setdefault("classifier", classifier) + return CSPClassifier(random_state=random_state, **kwargs) else: raise ValueError(f"UNKNOWN CLASSIFIER: {c} in get_classifier_by_name") diff --git a/tsml_eval/experiments/tests/test_classification.py b/tsml_eval/experiments/tests/test_classification.py index 9cdf5c1c4..cb05e3a0e 100644 --- a/tsml_eval/experiments/tests/test_classification.py +++ b/tsml_eval/experiments/tests/test_classification.py @@ -222,7 +222,7 @@ def test_get_classifier_by_name_invalid(): ("DeepConvNet", "DeepConvNetClassifier"), ("RiemannianMDM", "RiemannianMDMClassifier"), ("RiemannianKNN", "RiemannianKNNClassifier"), - ("CSPSVM", "CSPSVMClassifier"), + ("CSP", "CSPClassifier"), ], ) def test_get_aeon_neuro_classifier(name, expected): @@ -236,6 +236,28 @@ def test_get_aeon_neuro_classifier(name, expected): assert type(classifier).__name__ == expected +@pytest.mark.parametrize( + "name, expected", + [ + ("CSPRandomForest", "RandomForestClassifier"), + ("CSPRF", "RandomForestClassifier"), + ("CSPSVM", "SVC"), + ("CSPSVMClassifier", "SVC"), + ], +) +def test_get_aeon_neuro_csp_variants(name, expected): + """CSP variants use the requested sklearn classifier.""" + pytest.importorskip("aeon_neuro") + classifier = get_classifier_by_name(name, random_state=42, n_jobs=2) + assert type(classifier).__name__ == "CSPClassifier" + assert type(classifier.classifier).__name__ == expected + assert classifier.random_state == 42 + if expected == "RandomForestClassifier": + assert classifier.classifier.n_jobs == 2 + else: + assert classifier.classifier.probability + + def test_get_aeon_neuro_classifier_invalid(): """Test that the prefixed lookup only accepts exported neuro classifiers.""" pytest.importorskip("aeon_neuro")