Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.9", "3.11"]
python-version: ["3.11"]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
Expand Down
86 changes: 86 additions & 0 deletions benchmarks/pr10/ab_pipeline.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
"""Run the MPACT pipeline on the bundled PTY087I2 example with one code tree
and dump which features each filter removed, plus timings, as JSON.

Run once per code tree / variant, then compare the JSON files:

python ab_pipeline.py <repo root> <out dir> <variant>

variant: default | mdfil | adduct | both | relfil_beta | all
(the non-default variants only mean something on the PR tree)
"""
import importlib.util
import json
import sys
import time
from pathlib import Path

repo = Path(sys.argv[1]).resolve()
out = Path(sys.argv[2]).resolve()
variant = sys.argv[3] if len(sys.argv) > 3 else 'default'

sys.path.insert(0, str(repo / 'code'))
spec = importlib.util.spec_from_file_location(
'pipeline_fixture', repo / 'code' / 'tests' / 'test_msfast_pipeline.py')
fixture = importlib.util.module_from_spec(spec)
spec.loader.exec_module(fixture)

import filter as mpfilter # noqa: E402
import pandas as pd # noqa: E402
from MSFaST import run_MSFaST # noqa: E402

out.mkdir(parents=True, exist_ok=True)
params = fixture.make_params(out)
if variant in ('mdfil', 'both', 'all'):
params.mdfil = True
if variant in ('adduct', 'both', 'all'):
params.adductgrp = True
params.adduct_mode = 'pos'
if variant in ('relfil_beta', 'all'):
params.relfil_mode = 'beta'

timings = {}


def timed(name):
original = getattr(mpfilter, name, None)
if original is None:
return

def wrapper(*a, **k):
t0 = time.perf_counter()
try:
return original(*a, **k)
finally:
timings[name] = timings.get(name, 0.0) + time.perf_counter() - t0
setattr(mpfilter, name, wrapper)


for fn in ('relationalfilter', 'mdfilter', 'adductgroup', 'decon', 'cvfilter', 'applyfilters'):
timed(fn)

t0 = time.perf_counter()
result = run_MSFaST(params)
timings['total'] = time.perf_counter() - t0

stem = params.filename.stem
unfiltered = pd.read_csv(params.filename, sep=',', header=[0, 1, 2], index_col=[0, 1, 2])
filtered = pd.read_csv(params.outputdir / (stem + '_filtered.csv'),
sep=',', header=[0, 1, 2], index_col=[0, 1, 2])
iondict = pd.read_csv(params.outputdir / 'iondict.csv', index_col=0)

summary = {
'variant': variant,
'repo': str(repo),
'n_input': int(len(unfiltered.index)),
'n_filtered': int(len(filtered.index)),
'kept': [str(i[0]) for i in filtered.index],
'filters': {k: [str(x) for x in v.ions] for k, v in result.ionfilters.items()},
'pass_columns': {c: int((iondict[c].astype(str).str.upper() == 'TRUE').sum())
for c in iondict.columns if str(c).startswith('pass')},
'iondict_rows': int(len(iondict.index)),
'timings_s': {k: round(v, 3) for k, v in timings.items()},
'outputdir': str(params.outputdir),
}
(out / 'summary.json').write_text(json.dumps(summary, indent=1))
print(json.dumps({k: v for k, v in summary.items() if k not in ('kept', 'filters')}, indent=1))
print({k: len(v) for k, v in summary['filters'].items()})
53 changes: 53 additions & 0 deletions benchmarks/pr10/md_npatlas.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
"""False-rejection rate of the PR's mass-defect filter on known natural products.

Every NPAtlas compound is a real, published microbial natural product, so any
[M+H]+ (or [M+Na]+, [M-H]-) m/z that md_plausible() rejects is a real compound
the filter would delete.

python md_npatlas.py <repo root with the PR> <npatlas.tsv>
"""
import re
import sys
from pathlib import Path

import numpy as np
import pandas as pd

repo, npatlas = Path(sys.argv[1]).resolve(), Path(sys.argv[2])
sys.path.insert(0, str(repo / 'code'))
import filter as mpfilter # noqa: E402

DEFAULT = dict(hi_slope=0.00112, hi_int=0.03, lo_slope=-0.00025, lo_int=-0.03)
HALOGEN = dict(hi_slope=0.00112, hi_int=0.03, lo_slope=-0.0011, lo_int=-0.05)
PROTON, NA = 1.007276, 22.989218

df = pd.read_csv(npatlas, sep='\t', low_memory=False,
usecols=['compound_molecular_formula', 'compound_accurate_mass'])
df = df.dropna()
mass = df['compound_accurate_mass'].to_numpy(float)
formula = df['compound_molecular_formula'].astype(str)
halogen = formula.str.contains(r'Cl|Br|I(?![a-z])', regex=True).to_numpy()
elements = formula.map(lambda f: frozenset(re.findall(r'[A-Z][a-z]?', f)))
chnops = elements.map(lambda e: e <= {'C', 'H', 'N', 'O', 'P', 'S'}).to_numpy()

ions = {'[M+H]+': mass + PROTON, '[M+Na]+': mass + NA, '[M-H]-': mass - PROTON,
'[M+2H]2+': (mass + 2 * PROTON) / 2}

print(f'NPAtlas compounds: {len(mass)} (halogenated: {halogen.sum()}, CHNOPS-only: {chnops.sum()})')
print(f'{"ion":10} {"bounds":8} {"rejected":>9} {"%all":>7} {"%CHNOPS-only":>13} {"%halogenated":>13}')
for ion, mz in ions.items():
for label, b in (('default', DEFAULT), ('halogen', HALOGEN)):
ok = mpfilter.md_plausible(mz, **b)
rej = ~ok
print(f'{ion:10} {label:8} {rej.sum():9d} {100 * rej.mean():6.2f}% '
f'{100 * rej[chnops].mean():12.2f}% {100 * rej[halogen].mean():12.2f}%')

print('\n[M+H]+ rejection by mass range, default bounds:')
mz = ions['[M+H]+']
rej = ~mpfilter.md_plausible(mz, **DEFAULT)
for lo, hi in ((0, 200), (200, 300), (300, 400), (400, 500), (500, 600), (600, 700), (700, 5000)):
m = (mz >= lo) & (mz < hi)
print(f' {lo:4d}-{hi:<5d} n={m.sum():6d} rejected {100 * rej[m].mean():5.2f}%')

width_one = (1 - (DEFAULT['hi_int'] - DEFAULT['lo_int'])) / (DEFAULT['hi_slope'] - DEFAULT['lo_slope'])
print(f'\nFilter can reject nothing above m/z {width_one:.0f} (allowed window >= 1 Da wide).')
152 changes: 152 additions & 0 deletions benchmarks/pr10/msdial_reference.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
"""Benchmark the peak-correction and adduct code against MS-DIAL's own calls.

Uses the CHM350 Area_1 alignment (31,652 features, 48 injections), which exists
both as the raw MS-DIAL export (with MS-DIAL's isotope tracking and charge
assignment) and as the MPACT-layout table derived from it, row for row.

1. Isotopes: MS-DIAL marks 13C isotopologue rows ("Isotope tracking weight
number" > 0). Old and new relationalfilter are scored against that.
2. Adducts: target/decoy. adductgroup() is run with the real adduct mass
differences and then with shifted, chemically meaningless ones; decoy hits
estimate how many "adducts" are chance matches.

python msdial_reference.py <main repo> <PR repo> <CHM350_composite dir> <work dir>
"""
import importlib.util
import shutil
import sys
import time
from pathlib import Path
from types import SimpleNamespace

import numpy as np
import pandas as pd

main_repo, pr_repo, data_dir, work = (Path(a).resolve() for a in sys.argv[1:5])
STEM = 'Area_1_2024_11_19_19_38_54'


def load_filter(repo, name):
sys.path.insert(0, str(repo / 'code'))
spec = importlib.util.spec_from_file_location(name, repo / 'code' / 'filter.py')
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
sys.path.pop(0)
return mod


old = load_filter(main_repo, 'filter_main')
new = load_filter(pr_repo, 'filter_pr')

# ---- reference: MS-DIAL ---------------------------------------------------
ref = pd.read_csv(data_dir / (STEM + '.txt'), sep='\t', skiprows=4, low_memory=False)
table = pd.read_csv(data_dir / (STEM + '.csv'), header=[0, 1, 2], index_col=None, low_memory=False)
table.columns = pd.MultiIndex.from_tuples(
[('', '', c[2]) if i < 3 else c for i, c in enumerate(table.columns)])
ids = table.iloc[:, 0].astype(str).to_numpy()
mz = table.iloc[:, 1].to_numpy(float)
assert len(ref) == len(table) and np.allclose(mz, ref['Average Mz'].to_numpy(float), atol=1e-4), \
'MPACT table and MS-DIAL export are not row-aligned'
assert len(set(ids)) == len(ids), 'duplicate compound IDs'

weight = ref['Isotope tracking weight number'].to_numpy(int)
parent = ref['Isotope tracking parent ID'].to_numpy(int)
iso = weight > 0
dm = mz[iso] - ref['Average Mz'].to_numpy(float)[parent[iso]]
per_step = dm / weight[iso]
print(f'MS-DIAL: {len(ref)} features, {iso.sum()} marked as isotopologues')
print(f' sanity: mass step per isotope number, median {np.median(per_step):.4f} '
f'(1.0034 = 13C, 0.5017 = doubly charged); '
f'{np.mean(np.abs(per_step - 1.00336) < 0.01) * 100:.1f}% within 0.01 of 13C, '
f'{np.mean(np.abs(per_step - 0.50168) < 0.01) * 100:.1f}% within 0.01 of 13C/2')
iso_ids = set(ids[iso])
doubly = set(ids[(ref['Adduct type'] == '[M+2H]2+').to_numpy()])


# ---- build the pipeline's intermediate files ------------------------------
def group_of(injection):
name = injection.split('_', 1)[1]
if name.lower().startswith('blank'):
return 'Blank'
return name.rsplit('_r', 1)[0]


def make_outdir(tag):
out = work / tag
if out.exists():
shutil.rmtree(out)
out.mkdir(parents=True)
t = table.copy()
t.columns = pd.MultiIndex.from_tuples(
[(f'Unnamed: {i}_level_0', f'Unnamed: {i}_level_1', c[2]) if i < 3
else (group_of(c[2]), group_of(c[2]), c[2]) for i, c in enumerate(t.columns)])
t.to_csv(out / (STEM + '_formatted.csv'), index=False)
iondict = pd.DataFrame({'Compound': ids, 'm/z': mz, 'Retention time (min)': table.iloc[:, 2].to_numpy(float)})
iondict.to_csv(out / 'iondict.csv', index=False)
return out


def params_for(out, rtwin, mode='legacy'):
return SimpleNamespace(relfil_mode=mode, outputdir=out, filename=Path(STEM + '.csv'), RTwin=rtwin,
maxisowin=3, ringingwin=0.5, isopeakwin=0.01, dimerpeakwin=0.01,
adduct_mode='pos')


def score(label, removed, seconds):
removed = set(map(str, removed))
tp = len(removed & iso_ids)
prec = tp / len(removed) if removed else float('nan')
print(f' {label:26} removed {len(removed):5d} | isotopologues caught {tp:4d}/{len(iso_ids)} '
f'(recall {100 * tp / len(iso_ids):5.1f}%) | share of removals that are MS-DIAL isotopes '
f'{100 * prec:5.1f}% | {seconds:6.1f}s')
return removed


print('\n== 1. relationalfilter vs MS-DIAL isotope tracking ==')
for rtwin in (0.005, 0.02):
print(f' RT window {rtwin} min')
for label, mod, mode in (('main (published)', old, 'legacy'), ('PR, legacy mode', new, 'legacy'), ('PR, beta mode', new, 'beta')):
out = make_outdir(f'relfil_{label.split()[0]}_{mode}_{rtwin}')
t0 = time.perf_counter()
res = mod.relationalfilter(params_for(out, rtwin, mode), {})
score(label, res['relfil'].ions, time.perf_counter() - t0)

# ---- 2. adduct grouping: target vs decoy ----------------------------------
print('\n== 2. adductgroup target/decoy (RT window 0.02 min, 3 ppm) ==')
H, NA, K, NH4, H2O = 1.007276, 22.989770, 38.963707, 18.033823, 18.010565
TARGET = [('M+Na', NA - H), ('M+K', K - H), ('M+NH4', NH4 - H), ('M-H2O+H', -H2O), ('M-H4O2+H', -2 * H2O)]


def run_adduct(tag, adduct_list):
out = make_outdir(tag)
p = params_for(out, 0.02)
t0 = time.perf_counter()
new.adductgroup(p, adduct_list)
seconds = time.perf_counter() - t0
iondict = pd.read_csv(out / 'iondict.csv', index_col=0)
forms = iondict['adduct_form'].fillna('').astype(str)
counts = forms[forms.isin([n for n, _ in adduct_list])].value_counts()
return counts.reindex([n for n, _ in adduct_list]).fillna(0).astype(int), seconds, iondict


target_counts, seconds, target_iondict = run_adduct('adduct_target', TARGET)
print(f' target: {int(target_counts.sum())} ions assigned an adduct form in {seconds:.1f}s')

rng = np.random.default_rng(0)
decoys = []
for k in range(10):
shift = rng.uniform(0.15, 0.45, size=len(TARGET)) * rng.choice([-1, 1], size=len(TARGET))
counts, _, _ = run_adduct('adduct_decoy', [(n, d + s) for (n, d), s in zip(TARGET, shift)])
decoys.append(counts.to_numpy())
decoys = np.array(decoys)
print(f' {"form":10} {"target":>7} {"decoy mean":>11} {"decoy sd":>9} {"est. FDR":>9}')
for i, (name, _) in enumerate(TARGET):
t, d = target_counts.iloc[i], decoys[:, i].mean()
print(f' {name:10} {t:7d} {d:11.1f} {decoys[:, i].std():9.1f} {100 * min(1, d / t) if t else float("nan"):8.1f}%')
tot_t, tot_d = target_counts.sum(), decoys.sum(axis=1).mean()
print(f' {"all":10} {tot_t:7d} {tot_d:11.1f} {decoys.sum(axis=1).std():9.1f} {100 * tot_d / tot_t:8.1f}%')

# how often does an "adduct" call land on something MS-DIAL says is an isotope or 2+?
forms = target_iondict['adduct_form'].fillna('').astype(str)
called = set(forms[forms.isin([n for n, _ in TARGET])].index.astype(str))
print(f' adduct calls that MS-DIAL marks as isotopologues: {len(called & iso_ids)}, as [M+2H]2+: {len(called & doubly)}')
85 changes: 85 additions & 0 deletions benchmarks/pr10/relfil_pairs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
"""What does the peak-correction filter actually remove on PTY087I2?

Runs relationalfilter from one code tree on the example dataset, then describes
every (parent, removed feature) pair: mass difference, intensity ratio, and
whether the mass-defect filter would have removed the feature anyway.

python relfil_pairs.py <repo to take relationalfilter from> <repo with md_plausible> <work dir> [mode]

mode is passed as analysis_params.relfil_mode (legacy | beta) when given.
"""
import importlib.util
import sys
from pathlib import Path

import numpy as np
import pandas as pd

repo, pr_repo, work = (Path(a).resolve() for a in sys.argv[1:4])
mode = sys.argv[4] if len(sys.argv) > 4 else None

sys.path.insert(0, str(repo / 'code'))
spec = importlib.util.spec_from_file_location(
'pipeline_fixture', repo / 'code' / 'tests' / 'test_msfast_pipeline.py')
fixture = importlib.util.module_from_spec(spec)
spec.loader.exec_module(fixture)
import filter as mpfilter # noqa: E402
from MSFaST import run_MSFaST # noqa: E402

spec = importlib.util.spec_from_file_location('filter_pr', pr_repo / 'code' / 'filter.py')
prfilter = importlib.util.module_from_spec(spec)
spec.loader.exec_module(prfilter)

import shutil # noqa: E402
if work.exists():
shutil.rmtree(work)
work.mkdir(parents=True)
params = fixture.make_params(work)
if mode:
params.relfil_mode = mode

captured = {}
original = mpfilter.relationalfilter


def capture(analysis_params, ionfilterlist):
f = analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv')
captured['table'] = pd.read_csv(f, header=[0, 1, 2], index_col=0, low_memory=False)
out = original(analysis_params, ionfilterlist)
captured['merge'] = {k: list(np.atleast_1d(v.sources)) for k, v in out['relfil'].merge.items()}
return out


mpfilter.relationalfilter = capture
run_MSFaST(params)

t = captured['table']
t.index = t.index.astype(str)
mz = pd.to_numeric(t.iloc[:, 0]).to_dict()
keep = [c for c in t.columns[2:] if 'BLANK' not in str(c[0]).upper()]
mx = t[keep].apply(pd.to_numeric, errors='coerce').fillna(0).max(axis=1).to_dict()

rows = []
for parent, kids in captured['merge'].items():
for kid in kids:
p, k = str(parent), str(kid)
rows.append((k, mz[k] - mz[p], mx[k] / mx[p] if mx[p] > 0 else np.inf, mz[k]))
d = pd.DataFrame(rows, columns=['removed', 'dm', 'ratio', 'mz'])
frac = d['dm'] - np.floor(d['dm'])
n13 = np.round(d['dm'] / 1.00335)
iso = (n13 >= 1) & ((d['dm'] - n13 * 1.00335).abs() < 0.01)
half = (frac - 0.5017).abs() < 0.01
near = d['dm'] < 0.5
d['kind'] = np.select([iso, half, near], ['13C isotope spacing (n x 1.0034 +/- 0.01)',
'half-integer spacing (2+ ion)',
'within 0.5 Da of parent (ringing band)'],
'other spacing in 1.0-1.5 / 2.0-2.5 Da band')
d['md_fail'] = ~prfilter.md_plausible(d['mz'].to_numpy(), 0.00112, 0.03, -0.00025, -0.03)

print(f'\n{len(d)} features removed')
print(f'{"kind":46} {"n":>4} {"weaker than parent":>19} {"<1% of parent":>14} {"fails mass defect":>18}')
for kind, g in d.groupby('kind'):
print(f'{kind:46} {len(g):4d} {int((g.ratio < 1).sum()):19d} {int((g.ratio < 0.01).sum()):14d} {int(g.md_fail.sum()):18d}')
print(f'{"all":46} {len(d):4d} {int((d.ratio < 1).sum()):19d} {int((d.ratio < 0.01).sum()):14d} {int(d.md_fail.sum()):18d}')
print('removed feature STRONGER than the parent it was merged into:', int((d.ratio > 1).sum()))
print('intensity ratio removed/parent, quartiles:', np.percentile(d.ratio.replace(np.inf, 1e9), [10, 25, 50, 75, 90]).round(3))
Loading
Loading