Chore/modernise and cleanup

.github/workflows/test.yml

@@ -6,6 +6,16 @@ on:
   pull_request:
     branches: [main, "release/*", "dev"]
 
+env:
+  # Bump these together. torch-scatter/torch-sparse are only published for a
+  # subset of torch releases: https://data.pyg.org/whl/ has no wheels beyond
+  # torch 2.12, so raising TORCH_VERSION past it means those extensions would
+  # have to build from source, which fails on macOS (PyTorch's own
+  # c10/util/strong_type.h specializes std::is_arithmetic, which recent libc++
+  # rejects). Check the index before changing this.
+  TORCH_VERSION: "2.12.0"
+  PYTHON_VERSION: "3.12"
+
 jobs:
   run-tests:
     name: Pytest (${{ matrix.os }})
@@ -20,39 +30,63 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
-      - name: Set up conda env (spatial-pinned)
-        uses: conda-incubator/setup-miniconda@v3
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          activate-environment: spatial
-          environment-file: environment.ci-spatial.yaml
-          auto-activate: false
-          mamba-version: "*"
-          channels: pyg,pytorch,conda-forge,defaults
-          channel-priority: flexible
-
-      - name: Show Python and key package versions
-        shell: bash -el {0}
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+
+      # Linux resolves CUDA builds by default, which are ~3GB and useless on a
+      # CPU runner. macOS wheels on PyPI are already CPU/MPS only, and the
+      # download.pytorch.org cpu index does not publish macOS builds.
+      - name: Install PyTorch (CPU)
+        run: |
+          python -m pip install --upgrade pip
+          if [ "$RUNNER_OS" = "Linux" ]; then
+            pip install "torch==${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/cpu
+          else
+            pip install "torch==${TORCH_VERSION}"
+          fi
+
+      # Prebuilt binaries. Never let these build from source; see the note on
+      # TORCH_VERSION above.
+      - name: Install PyG compiled extensions
+        run: |
+          pip install torch-scatter torch-sparse \
+            --only-binary :all: \
+            -f "https://data.pyg.org/whl/torch-${TORCH_VERSION}+cpu.html"
+
+      - name: Install the package
+        run: pip install ".[interpretability,io]" pytest
+
+      - name: Show key package versions
         run: |
           python --version
           python - <<'PY'
           import importlib
 
           checks = [
               "torch",
+              "torch_geometric",
+              "torch_scatter",
+              "torch_sparse",
               "lightning",
               "pytorch_lightning",
+              "torchmetrics",
               "omegaconf",
               "hydra",
+              "numpy",
+              "pandas",
+              "scipy",
+              "sklearn",
               "plotly",
-              "torch_geometric",
           ]
           for name in checks:
               mod = importlib.import_module(name)
               print(f"{name}=={getattr(mod, '__version__', 'unknown')}")
           PY
 
       - name: Run pytest
-        shell: bash -el {0}
         run: pytest -v
 
   code-coverage:
@@ -63,20 +97,23 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
-      - name: Set up conda env (spatial-pinned)
-        uses: conda-incubator/setup-miniconda@v3
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          activate-environment: spatial
-          environment-file: environment.ci-spatial.yaml
-          auto-activate: false
-          mamba-version: "*"
-          channels: pyg,pytorch,conda-forge,defaults
-          channel-priority: flexible
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+
+      - name: Install PyTorch (CPU) and PyG extensions
+        run: |
+          python -m pip install --upgrade pip
+          pip install "torch==${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/cpu
+          pip install torch-scatter torch-sparse \
+            --only-binary :all: \
+            -f "https://data.pyg.org/whl/torch-${TORCH_VERSION}+cpu.html"
 
       - name: Run tests and collect coverage
-        shell: bash -el {0}
         run: |
-          pip install pytest-cov[toml]
+          pip install ".[interpretability,io]" pytest "pytest-cov[toml]"
           pytest --cov grass_mil
 
       - name: Upload coverage to Codecov
@@ -91,14 +128,18 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
+      # Deliberately the oldest supported Python, so `requires-python` is
+      # exercised at its floor rather than only at the version CI happens to use.
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
           python-version: "3.11"
+          cache: pip
 
-      # A plain pip install, deliberately outside conda: this is the path a
-      # downstream user takes, and it is the one that silently rots when the
-      # package layout or the config tree moves.
+      # A plain, unpinned pip install: this is the path a downstream user takes,
+      # and it is the one that silently rots when the package layout or the
+      # config tree moves. It deliberately does NOT pin torch, so a dependency
+      # that breaks on the latest release shows up here first.
       - name: Install the package
         run: pip install .
 

.gitignore

@@ -151,3 +151,6 @@ configs/local/default.yaml
 
 # Aim logging
 .aim
+
+# Vignette scratch space (downloaded data, graphs, runs)
+vignettes/vignette_work/

README.md

@@ -102,6 +102,7 @@ grass-mil-eval \
 
 ## Documentation Index
 
+- Worked examples on a public dataset: [`vignettes/`](vignettes/README.md)
 - Full tracked-file map: [`docs/repository_map.md`](docs/repository_map.md)
 - Dataset onboarding and preparation checklist: [`docs/dataset_preparation.md`](docs/dataset_preparation.md)
 - End-to-end workflows (new data, SSL pretrain->finetune, model adjustments, inference): [`docs/workflows_train_infer.md`](docs/workflows_train_infer.md)

docs/capabilities_and_modes.md

@@ -167,9 +167,10 @@ Validation (during `src/grass_mil/train.py`) includes:
 
 Notebooks support interactive smoke validation of:
 
-- data loading and graph construction (`notebooks/data_loading_testing.ipynb`)
-- component compatibility matrix (`notebooks/model_components_testing.ipynb`)
-- training regime smoke behavior (`notebooks/training_logic_testing.ipynb`)
+- ingesting a public AnnData cohort and building cellular graphs
+  (`vignettes/01_anndata_to_spatial_graphs.ipynb`)
+- self-supervised pretraining and spatial-niche discovery
+  (`vignettes/02_pretraining_and_niches.ipynb`)
 
 ## 12) CI And Quality Modes
 

docs/interpretability_suite.md

@@ -286,30 +286,41 @@ Plugin registration is per pipeline run (fresh `PluginRegistry`).
 
 Built-in plugins:
 
-- `niche_profiles`
-- `margin_attribution`
-- `neighborhood_enrichment`
-- `diff_neighborhood_enrichment`
-- `per_niche_cell_type_enrichment`
-- `filtration_curves`
-- `morans_i`
-- `ripley`
-- `niche_agreement`
-- `tissue_graph`
-
-Four of these are not enabled by any shipped preset because each needs an input
-the pipeline cannot infer:
+| Plugin | Unit |
+|---|---|
+| `niche_profiles` | instances, composition over all their cells |
+| `margin_attribution` | instances |
+| `cell_type_enrichment_per_niche` | cells, within each niche |
+| `cell_type_moran_per_niche` | cells, within each niche |
+| `cell_filtration_curves` | cell-cell edges, within each niche |
+| `cell_type_ripley_per_niche` | cells, within each niche |
+| `cell_type_diff_enrichment_by_condition` | cells, differenced between conditions |
+| `niche_label_moran` | instances, globally |
+| `niche_agreement` | instances |
+| `tissue_graph` | instances (visualisation) |
+
+The tier-2 statistics are cell-level: the instance-level versions were removed,
+because treating a whole ego-graph as one node measures how neighbourhood
+*labels* relate rather than how cells do. Two niches with identical composition
+but opposite spatial arrangement are indistinguishable to an instance-level
+analysis.
+
+The cell-level plugins need `cell_table` and `cell_edge_table`, exported by
+`grass-mil-predict interpretability.cells.enabled=true` and passed to the report
+as `data.cell_table` / `data.cell_edge_table`.
+
+Three plugins are not in any shipped preset, because each needs an input the
+pipeline cannot infer:
 
 | Plugin | Additional input required |
 |---|---|
-| `diff_neighborhood_enrichment` | a condition column, plus a permutation group column in permutation mode |
+| `cell_type_diff_enrichment_by_condition` | a condition column, plus a permutation group column in permutation mode |
 | `tissue_graph` | an explicit `sample_value` (no auto-pick) |
-| `niche_agreement` | a second niche labeling column to compare against |
-| `per_niche_cell_type_enrichment` | enabled in `full.yaml`; needs `cell_type` |
+| `niche_agreement` | a second niche labelling column to compare against |
 
-`filtration_curves` default threshold grid:
+`cell_filtration_curves` default threshold grid:
 
-- notebook-parity default is `np.linspace(0.0, 55.0, 500)`
+- `np.linspace(0.0, 55.0, 500)`
 - thresholds are interpreted in micrometers
 
 `tissue_graph` behavior:
@@ -535,8 +546,7 @@ Subgroups:
 - `configs/interpretability/clustering/*.yaml`
 - `configs/interpretability/plugins/default.yaml` — `niche_profiles` only
 - `configs/interpretability/plugins/full.yaml` — composes `default` and adds
-  `margin_attribution`, `neighborhood_enrichment`,
-  `per_niche_cell_type_enrichment`, `filtration_curves`, `morans_i`, `ripley`
+  `margin_attribution`, the four cell-level analyses, and `niche_label_moran`
 
 Select a preset with:
 
@@ -686,3 +696,55 @@ Outputs use the language a biologist reads, not the algorithm's:
 The *algorithm* is still clustering, so `run_clustering`, `ClusteringResult` and
 the sklearn parameters (`n_clusters`, `min_cluster_size`) keep their names. The
 distinction is deliberate: those are the method, the niches are the result.
+
+## What the tier-2 statistics are computed over
+
+Every tier-2 analysis runs on the **instance graph**, not on the cell graph.
+Confusing the two makes the results easy to over-read, so be explicit about it:
+
+- A row of `instance_table` is one **k-hop ego-graph**, rooted at one cell.
+- An edge in `spatial_table` connects two **instances**, i.e. two roots.
+- `comp_*` are the cell-type fractions among **all cells inside** that
+  ego-graph. This is the only place the full cell content survives.
+- `cell_type` is the **root cell's own type**, so that the instance graph can be
+  treated as a (sampled) cell graph.
+- `center_x`/`center_y` are the **mean position of all cells** in the ego-graph,
+  not the root cell's coordinates.
+
+What each analysis therefore measures:
+
+| Analysis | Unit | Measures |
+|---|---|---|
+| `cell_type_enrichment_per_niche` | cell pairs inside a niche | how often two cell types are in contact, versus chance |
+| `cell_type_moran_per_niche` | cells inside a niche | whether each cell type forms contiguous patches |
+| `cell_filtration_curves` | cell-cell edges inside a niche | cells of each type reached as the distance threshold grows |
+| `cell_type_ripley_per_niche` | cells inside a niche | cross-L between cell types, from their own coordinates |
+| `niche_label_moran` | instances, globally | whether niches form contiguous territories |
+| `niche_profiles` | instances | mean `comp_*` per niche, which reflects every cell in the ego-graphs |
+
+**Roots are a weighted sample, not every cell.** With
+`data.sampler.runtime.proportional_root_sampling=true` (the default) roots are
+drawn with inverse-frequency weighting by `property_name`, so rare cell types
+are deliberately over-represented and one cell can root several instances. Any
+cell-type distribution over instances therefore reflects that sampling design
+rather than the tissue. Set `proportional_root_sampling=false` for an unweighted
+draw.
+
+### Cell-level analyses
+
+Export `cell_table` and `cell_edges` with
+`grass-mil-predict interpretability.cells.enabled=true`, pass them to the report
+as `data.cell_table` / `data.cell_edge_table`, and these become available:
+
+| Analysis | Unit | Measures |
+|---|---|---|
+| `cell_type_enrichment_per_niche` | cell pairs within a niche | cell-type x cell-type contact enrichment, using only edges whose both endpoints lie in the niche; optionally differenced against Background |
+| `cell_type_moran_per_niche` | cells within a niche | Moran's I of each cell-type indicator, as a niche x cell-type table |
+| `cell_filtration_curves` | cell pairs within a niche | cells of each type reached as the edge-distance threshold grows, counting both endpoints |
+| `cell_type_ripley_per_niche` | cells within a niche | centred cross-L between cell types, from the cells' own coordinates |
+| `niche_label_moran` | instances, globally | whether niches form contiguous territories rather than interleaving |
+
+Each cell inherits the niche of the ego-graph it was sampled into, so a cell
+appearing in several overlapping ego-graphs contributes to each. This mirrors
+the pooling the legacy NSCLC reports do.
+

docs/repository_map.md

@@ -13,15 +13,14 @@ Scope rules used here:
 - `.project-root`: Root marker used by `rootutils.setup_root(...)`.
 - `.gitignore`: Ignore rules for generated artifacts, envs, logs, and data outputs.
 - `.env.example`: Example environment variable file loaded by Hydra/OmegaConf env references.
-- `.pre-commit-config.yaml`: Code quality hooks (format/lint/notebook cleanup/docs checks).
+- `.pre-commit-config.yaml`: Code quality hooks (ruff lint + format, plus basic file hygiene checks).
 - `LICENSE`: Project license text.
 - `README.md`: Primary entrypoint documentation and navigation index.
 - `Makefile`: Convenience targets for clean, format, tests, and default training.
 - `pyproject.toml`: Package metadata, dependencies, optional extras, console
   entry points (`grass-mil-train`, `-eval`, `-loocv`, `-predict`, `-report`),
   and pytest/coverage/ruff configuration.
-- `environment.yaml`: Conda environment alternative with pinned major versions.
-- `environment.ci-spatial.yaml`: Fully pinned environment used by CI.
+- `environment.yaml`: Conda environment alternative for local development, pinned to major versions. CI does not use it; the CI stack is pinned inline in `.github/workflows/test.yml`.
 
 ## CI/DevOps And Contribution Files
 
@@ -32,7 +31,7 @@ Scope rules used here:
 - `.github/workflows/code-quality-main.yaml`: Pre-commit quality checks on pushes to `main`.
 - `.github/workflows/code-quality-pr.yaml`: Pre-commit quality checks on changed PR files.
 - `.github/workflows/release-drafter.yml`: Automated draft release workflow.
-- `.github/workflows/test.yml`: Multi-OS, multi-Python automated test workflow.
+- `.github/workflows/test.yml`: Multi-OS (Linux/macOS) test workflow, plus coverage and a from-source packaging job.
 
 ## Rules And Contracts
 
@@ -84,9 +83,6 @@ Sampler subgroup:
 ### Experiment Config Group
 
 - `configs/experiment/debug.yaml`: Dummy-data composition for notebook/debug experiments.
-- `configs/experiment/notebook_data_shadow_custom.yaml`: Notebook data-loading scenario with shadow sampler.
-- `configs/experiment/notebook_model_components.yaml`: Notebook scenario for component-combination checks.
-- `configs/experiment/notebook_training_logic.yaml`: Notebook scenario for training-regime smoke tests.
 
 ### Extras/Hydra/Paths Groups
 
@@ -112,7 +108,7 @@ Root model module configs:
 
 - `configs/model/supervised_module.yaml`: Unified supervised module defaults (mean/MIL via task config).
 - `configs/model/bgrl_module.yaml`: BGRL pretraining module defaults.
-- `configs/model/components.yaml`: Component-instantiation playground config used in notebooks/tests.
+- `configs/model/components.yaml`: Component-instantiation config used by the component tests.
 
 Encoder presets:
 
@@ -412,12 +408,11 @@ Reporting:
 - `docs/interpretability_suite.md`: Interpretability architecture, contracts and plugin catalog.
 - `docs/interpretability_extension_guide.md`: How to add a new analysis plugin.
 
-### Notebooks
+### Vignettes
 
-- `notebooks/.gitkeep`: Placeholder to keep notebook directory tracked.
-- `notebooks/data_loading_testing.ipynb`: Interactive data-loading and graph-construction smoke notebook.
-- `notebooks/model_components_testing.ipynb`: Interactive model-component combinatorial smoke notebook.
-- `notebooks/training_logic_testing.ipynb`: Interactive training-regime smoke notebook.
+- `vignettes/README.md`: Index, dataset description and honest scope notes.
+- `vignettes/01_anndata_to_spatial_graphs.ipynb`: squidpy AnnData to manifest to cellular graphs.
+- `vignettes/02_pretraining_and_niches.ipynb`: BGRL pretraining, embedding export and niche discovery.
 
 ### Data/Logs Placeholders