diff --git a/_tsml_research_resources/soton/iridis/README.md b/_tsml_research_resources/soton/iridis/README.md index db3ecadcf..3d0aacb09 100644 --- a/_tsml_research_resources/soton/iridis/README.md +++ b/_tsml_research_resources/soton/iridis/README.md @@ -1,3 +1,11 @@ +eeg_classification_students.md: + Student guide for downloading aeon-neuro EEG problems and running aeon and + aeon-neuro classification experiments on Iridis 6. + +download_eeg_classification.py: + Downloads the available EEG classification problems with aeon-neuro and writes + a dataset list for the submission scripts. + serial_scripts: Simple(er) scripts which submit each experiment as a separate job and requests 1 core. This is fine for small numbers of experiments. diff --git a/_tsml_research_resources/soton/iridis/download_eeg_classification.py b/_tsml_research_resources/soton/iridis/download_eeg_classification.py new file mode 100644 index 000000000..f67e38c4a --- /dev/null +++ b/_tsml_research_resources/soton/iridis/download_eeg_classification.py @@ -0,0 +1,95 @@ +"""Download aeon-neuro EEG classification problems for tsml-eval experiments. + +Run this on an Iridis node with internet access. The output dataset list can be passed +to the Iridis classification submission scripts. +""" + +import argparse +import logging +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def main(): + """Download published problems and write a list of usable datasets.""" + logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-dir", type=Path, required=True) + parser.add_argument("--dataset-list", type=Path, required=True) + parser.add_argument( + "--only", nargs="+", metavar="NAME", help="Download selected datasets first" + ) + args = parser.parse_args() + + from aeon_neuro.datasets.classification_datasets import dataset_map, dataset_names + + names = args.only if args.only else dataset_names + unknown = sorted(set(names) - set(dataset_names)) + if unknown: + parser.error(f"Unknown aeon-neuro datasets: {', '.join(unknown)}") + + args.data_dir.mkdir(parents=True, exist_ok=True) + args.dataset_list.parent.mkdir(parents=True, exist_ok=True) + ready = [] + unavailable = [] + failed = [] + id_complete = [] + + for name in names: + problem_dir = args.data_dir / name + train = problem_dir / f"{name}_TRAIN.ts" + test = problem_dir / f"{name}_TEST.ts" + record_id = dataset_map.get(name) + if not (train.is_file() and test.is_file()): + if record_id in (None, 49): + logger.info("UNAVAILABLE %s (no download record yet)", name) + unavailable.append(name) + continue + + try: + # The loader downloads both splits; load only TRAIN to limit memory. + from aeon_neuro.datasets import load_eeg_classification + + load_eeg_classification( + name, split="train", extract_path=str(args.data_dir) + ) + if not train.is_file() or not test.is_file(): + raise FileNotFoundError( + f"Missing TRAIN or TEST file in {problem_dir}" + ) + except Exception as exc: + logger.info("FAILED %s: %s", name, exc) + failed.append(name) + continue + + if all( + (problem_dir / f"{name}_id_{split}.txt").is_file() + for split in ("TRAIN", "TEST") + ): + id_complete.append(name) + + logger.info("READY %s", name) + ready.append(name) + + args.dataset_list.write_text("".join(f"{name}\n" for name in ready)) + logger.info( + "Ready: %s; unavailable: %s; failed: %s; paired ID files: %s. " + "Dataset list: %s", + len(ready), + len(unavailable), + len(failed), + len(id_complete), + args.dataset_list, + ) + if unavailable: + logger.info("Unavailable: %s", ", ".join(unavailable)) + if failed: + logger.info("Failed: %s", ", ".join(failed)) + if unavailable or failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/_tsml_research_resources/soton/iridis/eeg_classification_students.md b/_tsml_research_resources/soton/iridis/eeg_classification_students.md new file mode 100644 index 000000000..aabc6467f --- /dev/null +++ b/_tsml_research_resources/soton/iridis/eeg_classification_students.md @@ -0,0 +1,226 @@ +# EEG classification experiments on Iridis 6 + +This guide takes you from a fresh Iridis account to tsml-eval prediction files for +classifiers from aeon and aeon-neuro. Commands below are for Bash on Iridis 6. The +[general Iridis Python guide](iridis_python.md) covers cluster access, storage, Conda, +and the supplied submission scripts in more detail. Check the [HPC Community Wiki](https://sotonac.sharepoint.com/teams/HPCCommunityWiki) +for current queue rules and account instructions. + +## Before you start + +You need an Iridis account, access to the Southampton network or VPN, and an +aeon-neuro release containing the EEG dataset downloader and classifiers. This guide +expects aeon-neuro **0.1.0 or later** and a tsml-eval checkout containing its `neuro` +extra. If the tsml-eval changes have not been merged into `main`, use the branch +provided by your supervisor. The code below uses Python 3.13 because the aeon-neuro +checkout inspected for this guide does not declare Python 3.14 support. + +The EEG dataset downloader depends on published download records. Some entries in +aeon-neuro's dataset map may still be marked `49` (unavailable). The download step +reports these names. **Do not submit experiments against an empty dataset list.** +Ask your supervisor for an updated aeon-neuro release or staged `.ts` files if this +happens. + +## 1. Connect and install the software + +Connect to `iridis6.soton.ac.uk` over SSH. Run these commands on the login node, +where Git and package downloads are available: + +```bash +module load conda/python3 +conda create -n tsml-neuro python=3.13 +conda activate tsml-neuro +git clone https://github.com/time-series-machine-learning/tsml-eval.git "$HOME/tsml-eval" +cd "$HOME/tsml-eval" +# If the aeon-neuro changes are not yet on main, checkout the supplied branch here. +python -m pip install --editable '.[neuro]' +python -c 'import aeon, aeon_neuro, tsml_eval; print(aeon.__version__, aeon_neuro.__version__, tsml_eval.__version__)' +``` + +If you intend to run aeon's TensorFlow classifiers such as +`InceptionTimeClassifier`, also install the `deep_learning` extra in the same +environment (or use a separate GPU environment): + +```bash +python -m pip install --editable '.[neuro,deep_learning]' +``` + +The editable install makes the Python files in your clone the ones that run. Keep the +clone at a known commit while collecting results; record `git rev-parse HEAD` and +`python -m pip freeze` alongside the results. + +## 2. Stage the EEG problems once + +Put datasets and results on your scratch storage, not inside the Git checkout. The +supplied Python script calls `load_eeg_classification` for each listed EEG problem, +checks that both `.ts` splits exist, and writes only usable dataset names to a text +file. At the aeon-neuro repository's main HEAD, that loader calls aeon's +`load_classification`, which downloads every file attached to the Zenodo record, +including paired `_id_TRAIN.txt` and `_id_TEST.txt` files when published. These +contain case or subject IDs in the archive. Each line in the dataset list is +consumed by the Iridis submission scripts. + +```bash +EEG_WORK="/iridisfs/scratch/$USER/eeg" +mkdir -p "$EEG_WORK/data" "$EEG_WORK/results" "$EEG_WORK/output" +python "$HOME/tsml-eval/_tsml_research_resources/soton/iridis/download_eeg_classification.py" \ + --data-dir "$EEG_WORK/data" \ + --dataset-list "$EEG_WORK/datasets.txt" +cat "$EEG_WORK/datasets.txt" +``` + +Run this download step on a node that has internet access. The [Iridis Python +guide](iridis_python.md) notes that compute nodes may not have internet access. The +script processes one problem at a time but does load its training split into memory; +for very large problems, ask the cluster team or your supervisor for a suitable +transfer method rather than running a heavy download on the login node. You can +check a single problem first with `--only SelfRegulationSCP1`. + +The resulting layout must be: + +```text +eeg/data/SelfRegulationSCP1/SelfRegulationSCP1_TRAIN.ts +eeg/data/SelfRegulationSCP1/SelfRegulationSCP1_TEST.ts +eeg/datasets.txt +``` + +Some problems also have `_id_TRAIN.txt` and `_id_TEST.txt` next to +their `.ts` files. The downloader reports how many complete ID pairs it found. +If the `.ts` files were staged before the current aeon download behavior, the loader +reuses them and does not fetch missing sidecars; stage those from their Zenodo record +if you need case IDs. +The `.ts` format preserves channel order and sometimes describes electrode names +in header comments, but the archive does not consistently provide a separate +channel-name file. Do not treat a channel index as a named electrode without +checking that dataset's header or original documentation. + +If some names are unavailable or a download fails, the script exits with an error +and prints a summary. It still writes the successfully staged names to +`datasets.txt`, so you can rerun it after the source problem is fixed. Existing +complete train/test pairs are reused. Inspect the list before submitting jobs. + +## 3. Run one smoke test + +First run one small experiment in an interactive compute session (`sinteractive` +on Iridis). Do this after the dataset files have been staged; compute nodes may not +be able to download them. In the new session, reactivate the environment: + +```bash +module load conda/python3 +conda activate tsml-neuro +EEG_WORK="/iridisfs/scratch/$USER/eeg" +python -u "$HOME/tsml-eval/tsml_eval/experiments/classification_experiments.py" \ + "$EEG_WORK/data" "$EEG_WORK/results" \ + ROCKET SelfRegulationSCP1 0 +``` + +Replace `SelfRegulationSCP1` with a name actually present in your +`datasets.txt`. `ROCKET` is an aeon classifier. For an aeon-neuro CPU classifier, +replace it with `RiemannianMDMClassifier` or `CSPClassifier` (CSP with LDA). +Use `CSPRandomForest` for CSP with a random forest or `CSPSVM` for CSP with a +linear SVM. The classifier +names and aliases implemented in this checkout are in +[`tsml_eval/experiments/_get_classifier.py`](../../../tsml_eval/experiments/_get_classifier.py). +New aeon-neuro classes exported by `aeon_neuro.classification` can be requested as +`neuro:ClassName`. + +The final positional argument `0` uses the supplied train/test split and is also +the result-file suffix. The file should appear at: + +```text +eeg/results/ROCKET/Predictions/SelfRegulationSCP1/testResample0.csv +``` + +The script skips a completed result by default. Add `-ow` only when you intend +to overwrite it. Add `-tr` if you also need a training results file; that can be +substantially slower because it may perform cross-validation. For nonzero resample +IDs, tsml-eval stratifies a new train/test split and writes, for example, +`testResample1.csv`. + +## 4. Submit CPU experiments across the dataset list + +The existing [serial classification submission script](serial_scripts/classification_experiments.sh) +submits Slurm array jobs for each dataset and classifier. Make a personal copy so +your paths and scheduler choices do not affect other students: + +```bash +cp "$HOME/tsml-eval/_tsml_research_resources/soton/iridis/serial_scripts/classification_experiments.sh" \ + "$HOME/eeg_classification_cpu.sh" +``` + +Edit the configuration block near the top of your copy. For a first pass, use one +fold (the original split), a modest submission limit, and your own paths: + +```bash +max_folds=1 +start_fold=1 +max_num_submitted=10 +queue="batch" +username="$USER" +local_path="$HOME/" +data_dir="/iridisfs/scratch/$USER/eeg/data/" +datasets="/iridisfs/scratch/$USER/eeg/datasets.txt" +results_dir="/iridisfs/scratch/$USER/eeg/results/" +out_dir="/iridisfs/scratch/$USER/eeg/output/" +script_file_path="$HOME/tsml-eval/tsml_eval/experiments/classification_experiments.py" +env_name="tsml-neuro" +classifiers_to_run="ROCKET RiemannianMDMClassifier" +generate_train_files="false" +predefined_folds="false" +normalise_data="false" +``` + +Keep the trailing `/` on directory settings; the script concatenates names onto +these paths. Check `max_memory` and `max_time` in the same block. Increase them +only if jobs show they need more resources. The script requests one CPU per task, +and the Python experiment script limits common thread pools to one. + +After checking that `datasets.txt` contains the intended names, leave the Conda +environment and run the **Bash** submission script on the login node: + +```bash +conda deactivate +cd "/iridisfs/scratch/$USER/eeg" +bash "$HOME/eeg_classification_cpu.sh" +squeue -u "$USER" +``` + +The script uses Bash-specific syntax, so run it with `bash`, not `sh`. It skips +result files that already exist and writes Slurm logs under `eeg/output/`. If +you need more resamples later, increase `max_folds`; fold 1 maps to result +`Resample0`, fold 2 to `Resample1`, and so on. + +## 5. GPU classifiers + +For aeon-neuro deep learning models such as `EEGNetClassifier` and +`DeepConvNetClassifier`, follow the [GPU environment instructions](iridis_python.md#51-tsml-eval-gpu) +to build an Apptainer sandbox with the required packages. Install this tsml-eval +checkout and its `neuro` extra **inside the sandbox**. Copy the +[GPU classification submission script](gpu_scripts/gpu_classification_experiments.sh) +and set its `data_dir`, `datasets`, `results_dir`, `out_dir`, +`script_file_path`, `container_path`, `username`, `classifiers_to_run`, and +fold and resource settings for your account. Set `classifiers_to_run` initially +to one model and one dataset. That script requests one GPU per array task on the +`gpu` partition and calls the same tsml-eval Python experiment file via +`apptainer exec --nv`. + +Confirm in a GPU job that the container can read the cloned source and scratch +data paths, imports `aeon_neuro`, and sees the GPU before submitting the full +dataset list. Do not assume that packages installed in your host Conda environment +are available inside the container. + +## 6. Check and keep the results + +Each successful test experiment writes a CSV under +`//Predictions//testResample.csv`. +If `-tr` was requested, there is also a `trainResample.csv`. The file begins +with experiment metadata and then stores predictions and probabilities. The +[results format notebook](../../../examples/results_format.ipynb) explains the +format and the [evaluation notebook](../../../examples/evaluation.ipynb) shows +how to compare classifiers. + +For a failed job, inspect the corresponding `.err` and `.out` files in +`eeg/output///`. If the submission script prints that a job +was skipped, check whether the expected result file already exists. Keep the +dataset list, the tsml-eval Git commit, package versions, and job settings with +the CSV files so experiments can be reproduced. diff --git a/docs/installation.md b/docs/installation.md index 9271cf777..3ab46aa67 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -25,6 +25,13 @@ required, or install all of them using the `all_extras` extra dependency set: pip install tsml-eval[all_extras] ``` +To run EEG classifiers from `aeon-neuro` 0.1.0 or later, install the `neuro` +extra with Python 3.12 or 3.13 (the Python versions supported by that release): + +```console +pip install tsml-eval[neuro] +``` + ```{note} If this results in a "no matches found" error, it may be due to how your shell handles special characters. Try surrounding the dependency portion with quotes i.e. @@ -32,6 +39,10 @@ pip install tsml-eval[all_extras] pip install tsml-eval"[all_extras]" ``` +Use an aeon-neuro classifier by its class name, for example +`get_classifier_by_name("EEGNetClassifier")`. Newly exported aeon-neuro +classifiers can also be selected as `neuro:ClassName`. + All extra dependency sets can be found in the [pyproject.toml](https://github.com/time-series-machine-learning/tsml-eval/blob/main/pyproject.toml) file `[project.optional-dependencies]` options. diff --git a/pyproject.toml b/pyproject.toml index 654eba69f..549bf6198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ [project.optional-dependencies] all_extras = [ "aeon[all_extras]", + "aeon-neuro>=0.1.0; python_version < '3.14'", "tsml[all_extras]", "xgboost", ] @@ -60,6 +61,9 @@ unstable_extras = [ deep_learning = [ "aeon[dl]", ] +neuro = [ + "aeon-neuro>=0.1.0; python_version < '3.14'", +] dev = [ "pre-commit", diff --git a/tsml_eval/experiments/_get_classifier.py b/tsml_eval/experiments/_get_classifier.py index 9d1e00d4d..129921f16 100644 --- a/tsml_eval/experiments/_get_classifier.py +++ b/tsml_eval/experiments/_get_classifier.py @@ -30,6 +30,15 @@ ["individualliteclassifier", "individuallite"], ["disjointcnnclassifier", "disjointcnn"], ] +neuro_classifiers = [ + ["eegnetclassifier", "eegnet"], + ["deepconvnetclassifier", "deepconvnet"], + ["riemannianmdmclassifier", "riemannianmdm"], + ["riemannianknnclassifier", "riemannianknn"], + ["cspclassifier", "csp", "csplda"], + ["csprandomforestclassifier", "csprandomforest", "csprf"], + ["cspsvmclassifier", "cspsvm"], +] dictionary_based_classifiers = [ ["bossensemble", "boss"], "individualboss", @@ -189,6 +198,8 @@ def get_classifier_by_name( return _set_classifier_interval_based( c, random_state, n_jobs, fit_contract, checkpoint, kwargs ) + elif str_in_nested_list(neuro_classifiers, c): + return _set_classifier_neuro(c, random_state, n_jobs, kwargs) elif str_in_nested_list(other_classifiers, c): return _set_classifier_other( c, random_state, n_jobs, fit_contract, checkpoint, kwargs @@ -201,6 +212,10 @@ def get_classifier_by_name( return _set_classifier_vector( c, random_state, n_jobs, fit_contract, checkpoint, kwargs ) + elif c.startswith("neuro:"): + return _set_classifier_neuro_by_class_name( + c.removeprefix("neuro:"), random_state, n_jobs, kwargs + ) else: raise ValueError(f"UNKNOWN CLASSIFIER: {c} in get_classifier_by_name") @@ -346,6 +361,80 @@ def _set_classifier_deep_learning( raise ValueError(f"UNKNOWN CLASSIFIER: {c} in get_classifier_by_name") +def _set_classifier_neuro(c, random_state, n_jobs, kwargs): + if c in ("eegnetclassifier", "eegnet"): + from aeon_neuro.classification.deep_learning import EEGNetClassifier + + return EEGNetClassifier(random_state=random_state, **kwargs) + elif c in ("deepconvnetclassifier", "deepconvnet"): + from aeon_neuro.classification.deep_learning import DeepConvNetClassifier + + return DeepConvNetClassifier(random_state=random_state, **kwargs) + elif c in ("riemannianmdmclassifier", "riemannianmdm"): + from aeon_neuro.classification.distance_based import RiemannianMDMClassifier + + return RiemannianMDMClassifier(n_jobs=n_jobs, **kwargs) + elif c in ("riemannianknnclassifier", "riemannianknn"): + from aeon_neuro.classification.distance_based import RiemannianKNNClassifier + + return RiemannianKNNClassifier(n_jobs=n_jobs, **kwargs) + elif c in ( + "cspclassifier", + "csp", + "csplda", + "csprandomforestclassifier", + "csprandomforest", + "csprf", + "cspsvmclassifier", + "cspsvm", + ): + from aeon_neuro.classification.feature_based import CSPClassifier + + if c in ("csprandomforestclassifier", "csprandomforest", "csprf"): + from sklearn.ensemble import RandomForestClassifier + + classifier = RandomForestClassifier( + random_state=random_state, n_jobs=n_jobs + ) + elif c in ("cspsvmclassifier", "cspsvm"): + from sklearn.svm import SVC + + classifier = SVC( + kernel="linear", probability=True, random_state=random_state + ) + else: + classifier = None + kwargs.setdefault("classifier", classifier) + return CSPClassifier(random_state=random_state, **kwargs) + else: + raise ValueError(f"UNKNOWN CLASSIFIER: {c} in get_classifier_by_name") + + +def _set_classifier_neuro_by_class_name(c, random_state, n_jobs, kwargs): + """Create an aeon-neuro classifier exported by its classification package.""" + from inspect import isclass, signature + + from aeon.classification.base import BaseClassifier + from aeon_neuro import classification + + for classifier in vars(classification).values(): + if ( + isclass(classifier) + and classifier.__name__.casefold() == c + and classifier.__module__.startswith("aeon_neuro.") + and issubclass(classifier, BaseClassifier) + ): + parameters = signature(classifier).parameters + init_kwargs = kwargs.copy() + if "random_state" in parameters: + init_kwargs.setdefault("random_state", random_state) + if "n_jobs" in parameters: + init_kwargs.setdefault("n_jobs", n_jobs) + return classifier(**init_kwargs) + + raise ValueError(f"UNKNOWN CLASSIFIER: neuro:{c} in get_classifier_by_name") + + def _set_classifier_dictionary_based( c, random_state, n_jobs, fit_contract, checkpoint, kwargs ): diff --git a/tsml_eval/experiments/tests/test_classification.py b/tsml_eval/experiments/tests/test_classification.py index 7a69451f1..cb05e3a0e 100644 --- a/tsml_eval/experiments/tests/test_classification.py +++ b/tsml_eval/experiments/tests/test_classification.py @@ -215,6 +215,85 @@ def test_get_classifier_by_name_invalid(): get_classifier_by_name("invalid") +@pytest.mark.parametrize( + "name, expected", + [ + ("EEGNet", "EEGNetClassifier"), + ("DeepConvNet", "DeepConvNetClassifier"), + ("RiemannianMDM", "RiemannianMDMClassifier"), + ("RiemannianKNN", "RiemannianKNNClassifier"), + ("CSP", "CSPClassifier"), + ], +) +def test_get_aeon_neuro_classifier(name, expected): + """Test native aeon-neuro classifiers are available by name and alias.""" + pytest.importorskip("aeon_neuro") + for classifier_name in (name, expected, f"neuro:{expected}"): + try: + classifier = get_classifier_by_name(classifier_name) + except ModuleNotFoundError as exc: + pytest.skip(str(exc)) + assert type(classifier).__name__ == expected + + +@pytest.mark.parametrize( + "name, expected", + [ + ("CSPRandomForest", "RandomForestClassifier"), + ("CSPRF", "RandomForestClassifier"), + ("CSPSVM", "SVC"), + ("CSPSVMClassifier", "SVC"), + ], +) +def test_get_aeon_neuro_csp_variants(name, expected): + """CSP variants use the requested sklearn classifier.""" + pytest.importorskip("aeon_neuro") + classifier = get_classifier_by_name(name, random_state=42, n_jobs=2) + assert type(classifier).__name__ == "CSPClassifier" + assert type(classifier.classifier).__name__ == expected + assert classifier.random_state == 42 + if expected == "RandomForestClassifier": + assert classifier.classifier.n_jobs == 2 + else: + assert classifier.classifier.probability + + +def test_get_aeon_neuro_classifier_invalid(): + """Test that the prefixed lookup only accepts exported neuro classifiers.""" + pytest.importorskip("aeon_neuro") + with pytest.raises(ValueError, match="UNKNOWN CLASSIFIER"): + get_classifier_by_name("neuro:InvalidClassifier") + + +def test_get_future_aeon_neuro_classifier(monkeypatch): + """Test discovery of a newly exported classifier and shared parameters.""" + classification = pytest.importorskip("aeon_neuro.classification") + from aeon.classification.base import BaseClassifier + + class FutureClassifier(BaseClassifier): + __module__ = "aeon_neuro.classification.future" + + def __init__(self, random_state=None, n_jobs=1): + self.random_state = random_state + self.n_jobs = n_jobs + super().__init__() + + def _fit(self, X, y): + return self + + def _predict(self, X): + return [0] * len(X) + + monkeypatch.setattr( + classification, "FutureClassifier", FutureClassifier, raising=False + ) + classifier = get_classifier_by_name( + "neuro:FutureClassifier", random_state=42, n_jobs=2 + ) + assert isinstance(classifier, FutureClassifier) + assert (classifier.random_state, classifier.n_jobs) == (42, 2) + + def test_aeon_classifiers_available(): """Test all aeon classifiers are available.""" excluded = [ diff --git a/tsml_eval/experiments/tests/test_download_eeg_classification.py b/tsml_eval/experiments/tests/test_download_eeg_classification.py new file mode 100644 index 000000000..9b98cdf43 --- /dev/null +++ b/tsml_eval/experiments/tests/test_download_eeg_classification.py @@ -0,0 +1,62 @@ +"""Tests for the Iridis EEG dataset download helper.""" + +import runpy +import sys +from pathlib import Path +from types import ModuleType + +import pytest + + +def test_download_eeg_classification_writes_only_ready_datasets(monkeypatch, tmp_path): + """Include staged and downloaded datasets, and report unavailable records.""" + data_dir = tmp_path / "data" + ready_dir = data_dir / "Ready" + ready_dir.mkdir(parents=True) + (ready_dir / "Ready_TRAIN.ts").write_text("train") + (ready_dir / "Ready_TEST.ts").write_text("test") + (ready_dir / "Ready_id_TRAIN.txt").write_text("subject-id\n") + (ready_dir / "Ready_id_TEST.txt").write_text("subject-id\n") + dataset_list = tmp_path / "datasets.txt" + + datasets = ModuleType("aeon_neuro.datasets") + datasets.__path__ = [] + collection = ModuleType("aeon_neuro.datasets.classification_datasets") + collection.dataset_names = ["Ready", "Download", "Unavailable"] + collection.dataset_map = {"Ready": 1, "Download": 2, "Unavailable": 49} + + def load_eeg_classification(name, split, extract_path): + assert (name, split) == ("Download", "train") + problem_dir = Path(extract_path) / name + problem_dir.mkdir() + (problem_dir / f"{name}_TRAIN.ts").write_text("train") + (problem_dir / f"{name}_TEST.ts").write_text("test") + + datasets.load_eeg_classification = load_eeg_classification + monkeypatch.setitem(sys.modules, "aeon_neuro.datasets", datasets) + monkeypatch.setitem( + sys.modules, "aeon_neuro.datasets.classification_datasets", collection + ) + monkeypatch.setattr( + sys, + "argv", + [ + "download_eeg_classification.py", + "--data-dir", + str(data_dir), + "--dataset-list", + str(dataset_list), + ], + ) + + script = ( + Path(__file__).parents[3] + / "_tsml_research_resources/soton/iridis/download_eeg_classification.py" + ) + with pytest.raises(SystemExit, match="1"): + runpy.run_path(str(script), run_name="__main__") + + assert dataset_list.read_text().splitlines() == ["Ready", "Download"] + assert (ready_dir / "Ready_id_TRAIN.txt").read_text() == "subject-id\n" + assert (ready_dir / "Ready_id_TEST.txt").read_text() == "subject-id\n" + assert not (data_dir / "Download" / "Download_id_TRAIN.txt").exists()