diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbf1c86..6271aa2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.9", "3.11"] + python-version: ["3.11"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/benchmarks/pr10/ab_pipeline.py b/benchmarks/pr10/ab_pipeline.py new file mode 100644 index 0000000..2938ffb --- /dev/null +++ b/benchmarks/pr10/ab_pipeline.py @@ -0,0 +1,86 @@ +"""Run the MPACT pipeline on the bundled PTY087I2 example with one code tree +and dump which features each filter removed, plus timings, as JSON. + +Run once per code tree / variant, then compare the JSON files: + + python ab_pipeline.py + +variant: default | mdfil | adduct | both | relfil_beta | all +(the non-default variants only mean something on the PR tree) +""" +import importlib.util +import json +import sys +import time +from pathlib import Path + +repo = Path(sys.argv[1]).resolve() +out = Path(sys.argv[2]).resolve() +variant = sys.argv[3] if len(sys.argv) > 3 else 'default' + +sys.path.insert(0, str(repo / 'code')) +spec = importlib.util.spec_from_file_location( + 'pipeline_fixture', repo / 'code' / 'tests' / 'test_msfast_pipeline.py') +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) + +import filter as mpfilter # noqa: E402 +import pandas as pd # noqa: E402 +from MSFaST import run_MSFaST # noqa: E402 + +out.mkdir(parents=True, exist_ok=True) +params = fixture.make_params(out) +if variant in ('mdfil', 'both', 'all'): + params.mdfil = True +if variant in ('adduct', 'both', 'all'): + params.adductgrp = True + params.adduct_mode = 'pos' +if variant in ('relfil_beta', 'all'): + params.relfil_mode = 'beta' + +timings = {} + + +def timed(name): + original = getattr(mpfilter, name, None) + if original is None: + return + + def wrapper(*a, **k): + t0 = time.perf_counter() + try: + return original(*a, **k) + finally: + timings[name] = timings.get(name, 0.0) + time.perf_counter() - t0 + setattr(mpfilter, name, wrapper) + + +for fn in ('relationalfilter', 'mdfilter', 'adductgroup', 'decon', 'cvfilter', 'applyfilters'): + timed(fn) + +t0 = time.perf_counter() +result = run_MSFaST(params) +timings['total'] = time.perf_counter() - t0 + +stem = params.filename.stem +unfiltered = pd.read_csv(params.filename, sep=',', header=[0, 1, 2], index_col=[0, 1, 2]) +filtered = pd.read_csv(params.outputdir / (stem + '_filtered.csv'), + sep=',', header=[0, 1, 2], index_col=[0, 1, 2]) +iondict = pd.read_csv(params.outputdir / 'iondict.csv', index_col=0) + +summary = { + 'variant': variant, + 'repo': str(repo), + 'n_input': int(len(unfiltered.index)), + 'n_filtered': int(len(filtered.index)), + 'kept': [str(i[0]) for i in filtered.index], + 'filters': {k: [str(x) for x in v.ions] for k, v in result.ionfilters.items()}, + 'pass_columns': {c: int((iondict[c].astype(str).str.upper() == 'TRUE').sum()) + for c in iondict.columns if str(c).startswith('pass')}, + 'iondict_rows': int(len(iondict.index)), + 'timings_s': {k: round(v, 3) for k, v in timings.items()}, + 'outputdir': str(params.outputdir), +} +(out / 'summary.json').write_text(json.dumps(summary, indent=1)) +print(json.dumps({k: v for k, v in summary.items() if k not in ('kept', 'filters')}, indent=1)) +print({k: len(v) for k, v in summary['filters'].items()}) diff --git a/benchmarks/pr10/md_npatlas.py b/benchmarks/pr10/md_npatlas.py new file mode 100644 index 0000000..a9b4ed8 --- /dev/null +++ b/benchmarks/pr10/md_npatlas.py @@ -0,0 +1,53 @@ +"""False-rejection rate of the PR's mass-defect filter on known natural products. + +Every NPAtlas compound is a real, published microbial natural product, so any +[M+H]+ (or [M+Na]+, [M-H]-) m/z that md_plausible() rejects is a real compound +the filter would delete. + + python md_npatlas.py +""" +import re +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +repo, npatlas = Path(sys.argv[1]).resolve(), Path(sys.argv[2]) +sys.path.insert(0, str(repo / 'code')) +import filter as mpfilter # noqa: E402 + +DEFAULT = dict(hi_slope=0.00112, hi_int=0.03, lo_slope=-0.00025, lo_int=-0.03) +HALOGEN = dict(hi_slope=0.00112, hi_int=0.03, lo_slope=-0.0011, lo_int=-0.05) +PROTON, NA = 1.007276, 22.989218 + +df = pd.read_csv(npatlas, sep='\t', low_memory=False, + usecols=['compound_molecular_formula', 'compound_accurate_mass']) +df = df.dropna() +mass = df['compound_accurate_mass'].to_numpy(float) +formula = df['compound_molecular_formula'].astype(str) +halogen = formula.str.contains(r'Cl|Br|I(?![a-z])', regex=True).to_numpy() +elements = formula.map(lambda f: frozenset(re.findall(r'[A-Z][a-z]?', f))) +chnops = elements.map(lambda e: e <= {'C', 'H', 'N', 'O', 'P', 'S'}).to_numpy() + +ions = {'[M+H]+': mass + PROTON, '[M+Na]+': mass + NA, '[M-H]-': mass - PROTON, + '[M+2H]2+': (mass + 2 * PROTON) / 2} + +print(f'NPAtlas compounds: {len(mass)} (halogenated: {halogen.sum()}, CHNOPS-only: {chnops.sum()})') +print(f'{"ion":10} {"bounds":8} {"rejected":>9} {"%all":>7} {"%CHNOPS-only":>13} {"%halogenated":>13}') +for ion, mz in ions.items(): + for label, b in (('default', DEFAULT), ('halogen', HALOGEN)): + ok = mpfilter.md_plausible(mz, **b) + rej = ~ok + print(f'{ion:10} {label:8} {rej.sum():9d} {100 * rej.mean():6.2f}% ' + f'{100 * rej[chnops].mean():12.2f}% {100 * rej[halogen].mean():12.2f}%') + +print('\n[M+H]+ rejection by mass range, default bounds:') +mz = ions['[M+H]+'] +rej = ~mpfilter.md_plausible(mz, **DEFAULT) +for lo, hi in ((0, 200), (200, 300), (300, 400), (400, 500), (500, 600), (600, 700), (700, 5000)): + m = (mz >= lo) & (mz < hi) + print(f' {lo:4d}-{hi:<5d} n={m.sum():6d} rejected {100 * rej[m].mean():5.2f}%') + +width_one = (1 - (DEFAULT['hi_int'] - DEFAULT['lo_int'])) / (DEFAULT['hi_slope'] - DEFAULT['lo_slope']) +print(f'\nFilter can reject nothing above m/z {width_one:.0f} (allowed window >= 1 Da wide).') diff --git a/benchmarks/pr10/msdial_reference.py b/benchmarks/pr10/msdial_reference.py new file mode 100644 index 0000000..54d6e5f --- /dev/null +++ b/benchmarks/pr10/msdial_reference.py @@ -0,0 +1,152 @@ +"""Benchmark the peak-correction and adduct code against MS-DIAL's own calls. + +Uses the CHM350 Area_1 alignment (31,652 features, 48 injections), which exists +both as the raw MS-DIAL export (with MS-DIAL's isotope tracking and charge +assignment) and as the MPACT-layout table derived from it, row for row. + + 1. Isotopes: MS-DIAL marks 13C isotopologue rows ("Isotope tracking weight + number" > 0). Old and new relationalfilter are scored against that. + 2. Adducts: target/decoy. adductgroup() is run with the real adduct mass + differences and then with shifted, chemically meaningless ones; decoy hits + estimate how many "adducts" are chance matches. + + python msdial_reference.py
+""" +import importlib.util +import shutil +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +main_repo, pr_repo, data_dir, work = (Path(a).resolve() for a in sys.argv[1:5]) +STEM = 'Area_1_2024_11_19_19_38_54' + + +def load_filter(repo, name): + sys.path.insert(0, str(repo / 'code')) + spec = importlib.util.spec_from_file_location(name, repo / 'code' / 'filter.py') + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + sys.path.pop(0) + return mod + + +old = load_filter(main_repo, 'filter_main') +new = load_filter(pr_repo, 'filter_pr') + +# ---- reference: MS-DIAL --------------------------------------------------- +ref = pd.read_csv(data_dir / (STEM + '.txt'), sep='\t', skiprows=4, low_memory=False) +table = pd.read_csv(data_dir / (STEM + '.csv'), header=[0, 1, 2], index_col=None, low_memory=False) +table.columns = pd.MultiIndex.from_tuples( + [('', '', c[2]) if i < 3 else c for i, c in enumerate(table.columns)]) +ids = table.iloc[:, 0].astype(str).to_numpy() +mz = table.iloc[:, 1].to_numpy(float) +assert len(ref) == len(table) and np.allclose(mz, ref['Average Mz'].to_numpy(float), atol=1e-4), \ + 'MPACT table and MS-DIAL export are not row-aligned' +assert len(set(ids)) == len(ids), 'duplicate compound IDs' + +weight = ref['Isotope tracking weight number'].to_numpy(int) +parent = ref['Isotope tracking parent ID'].to_numpy(int) +iso = weight > 0 +dm = mz[iso] - ref['Average Mz'].to_numpy(float)[parent[iso]] +per_step = dm / weight[iso] +print(f'MS-DIAL: {len(ref)} features, {iso.sum()} marked as isotopologues') +print(f' sanity: mass step per isotope number, median {np.median(per_step):.4f} ' + f'(1.0034 = 13C, 0.5017 = doubly charged); ' + f'{np.mean(np.abs(per_step - 1.00336) < 0.01) * 100:.1f}% within 0.01 of 13C, ' + f'{np.mean(np.abs(per_step - 0.50168) < 0.01) * 100:.1f}% within 0.01 of 13C/2') +iso_ids = set(ids[iso]) +doubly = set(ids[(ref['Adduct type'] == '[M+2H]2+').to_numpy()]) + + +# ---- build the pipeline's intermediate files ------------------------------ +def group_of(injection): + name = injection.split('_', 1)[1] + if name.lower().startswith('blank'): + return 'Blank' + return name.rsplit('_r', 1)[0] + + +def make_outdir(tag): + out = work / tag + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + t = table.copy() + t.columns = pd.MultiIndex.from_tuples( + [(f'Unnamed: {i}_level_0', f'Unnamed: {i}_level_1', c[2]) if i < 3 + else (group_of(c[2]), group_of(c[2]), c[2]) for i, c in enumerate(t.columns)]) + t.to_csv(out / (STEM + '_formatted.csv'), index=False) + iondict = pd.DataFrame({'Compound': ids, 'm/z': mz, 'Retention time (min)': table.iloc[:, 2].to_numpy(float)}) + iondict.to_csv(out / 'iondict.csv', index=False) + return out + + +def params_for(out, rtwin, mode='legacy'): + return SimpleNamespace(relfil_mode=mode, outputdir=out, filename=Path(STEM + '.csv'), RTwin=rtwin, + maxisowin=3, ringingwin=0.5, isopeakwin=0.01, dimerpeakwin=0.01, + adduct_mode='pos') + + +def score(label, removed, seconds): + removed = set(map(str, removed)) + tp = len(removed & iso_ids) + prec = tp / len(removed) if removed else float('nan') + print(f' {label:26} removed {len(removed):5d} | isotopologues caught {tp:4d}/{len(iso_ids)} ' + f'(recall {100 * tp / len(iso_ids):5.1f}%) | share of removals that are MS-DIAL isotopes ' + f'{100 * prec:5.1f}% | {seconds:6.1f}s') + return removed + + +print('\n== 1. relationalfilter vs MS-DIAL isotope tracking ==') +for rtwin in (0.005, 0.02): + print(f' RT window {rtwin} min') + for label, mod, mode in (('main (published)', old, 'legacy'), ('PR, legacy mode', new, 'legacy'), ('PR, beta mode', new, 'beta')): + out = make_outdir(f'relfil_{label.split()[0]}_{mode}_{rtwin}') + t0 = time.perf_counter() + res = mod.relationalfilter(params_for(out, rtwin, mode), {}) + score(label, res['relfil'].ions, time.perf_counter() - t0) + +# ---- 2. adduct grouping: target vs decoy ---------------------------------- +print('\n== 2. adductgroup target/decoy (RT window 0.02 min, 3 ppm) ==') +H, NA, K, NH4, H2O = 1.007276, 22.989770, 38.963707, 18.033823, 18.010565 +TARGET = [('M+Na', NA - H), ('M+K', K - H), ('M+NH4', NH4 - H), ('M-H2O+H', -H2O), ('M-H4O2+H', -2 * H2O)] + + +def run_adduct(tag, adduct_list): + out = make_outdir(tag) + p = params_for(out, 0.02) + t0 = time.perf_counter() + new.adductgroup(p, adduct_list) + seconds = time.perf_counter() - t0 + iondict = pd.read_csv(out / 'iondict.csv', index_col=0) + forms = iondict['adduct_form'].fillna('').astype(str) + counts = forms[forms.isin([n for n, _ in adduct_list])].value_counts() + return counts.reindex([n for n, _ in adduct_list]).fillna(0).astype(int), seconds, iondict + + +target_counts, seconds, target_iondict = run_adduct('adduct_target', TARGET) +print(f' target: {int(target_counts.sum())} ions assigned an adduct form in {seconds:.1f}s') + +rng = np.random.default_rng(0) +decoys = [] +for k in range(10): + shift = rng.uniform(0.15, 0.45, size=len(TARGET)) * rng.choice([-1, 1], size=len(TARGET)) + counts, _, _ = run_adduct('adduct_decoy', [(n, d + s) for (n, d), s in zip(TARGET, shift)]) + decoys.append(counts.to_numpy()) +decoys = np.array(decoys) +print(f' {"form":10} {"target":>7} {"decoy mean":>11} {"decoy sd":>9} {"est. FDR":>9}') +for i, (name, _) in enumerate(TARGET): + t, d = target_counts.iloc[i], decoys[:, i].mean() + print(f' {name:10} {t:7d} {d:11.1f} {decoys[:, i].std():9.1f} {100 * min(1, d / t) if t else float("nan"):8.1f}%') +tot_t, tot_d = target_counts.sum(), decoys.sum(axis=1).mean() +print(f' {"all":10} {tot_t:7d} {tot_d:11.1f} {decoys.sum(axis=1).std():9.1f} {100 * tot_d / tot_t:8.1f}%') + +# how often does an "adduct" call land on something MS-DIAL says is an isotope or 2+? +forms = target_iondict['adduct_form'].fillna('').astype(str) +called = set(forms[forms.isin([n for n, _ in TARGET])].index.astype(str)) +print(f' adduct calls that MS-DIAL marks as isotopologues: {len(called & iso_ids)}, as [M+2H]2+: {len(called & doubly)}') diff --git a/benchmarks/pr10/relfil_pairs.py b/benchmarks/pr10/relfil_pairs.py new file mode 100644 index 0000000..e27cf9f --- /dev/null +++ b/benchmarks/pr10/relfil_pairs.py @@ -0,0 +1,85 @@ +"""What does the peak-correction filter actually remove on PTY087I2? + +Runs relationalfilter from one code tree on the example dataset, then describes +every (parent, removed feature) pair: mass difference, intensity ratio, and +whether the mass-defect filter would have removed the feature anyway. + + python relfil_pairs.py [mode] + +mode is passed as analysis_params.relfil_mode (legacy | beta) when given. +""" +import importlib.util +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +repo, pr_repo, work = (Path(a).resolve() for a in sys.argv[1:4]) +mode = sys.argv[4] if len(sys.argv) > 4 else None + +sys.path.insert(0, str(repo / 'code')) +spec = importlib.util.spec_from_file_location( + 'pipeline_fixture', repo / 'code' / 'tests' / 'test_msfast_pipeline.py') +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) +import filter as mpfilter # noqa: E402 +from MSFaST import run_MSFaST # noqa: E402 + +spec = importlib.util.spec_from_file_location('filter_pr', pr_repo / 'code' / 'filter.py') +prfilter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(prfilter) + +import shutil # noqa: E402 +if work.exists(): + shutil.rmtree(work) +work.mkdir(parents=True) +params = fixture.make_params(work) +if mode: + params.relfil_mode = mode + +captured = {} +original = mpfilter.relationalfilter + + +def capture(analysis_params, ionfilterlist): + f = analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv') + captured['table'] = pd.read_csv(f, header=[0, 1, 2], index_col=0, low_memory=False) + out = original(analysis_params, ionfilterlist) + captured['merge'] = {k: list(np.atleast_1d(v.sources)) for k, v in out['relfil'].merge.items()} + return out + + +mpfilter.relationalfilter = capture +run_MSFaST(params) + +t = captured['table'] +t.index = t.index.astype(str) +mz = pd.to_numeric(t.iloc[:, 0]).to_dict() +keep = [c for c in t.columns[2:] if 'BLANK' not in str(c[0]).upper()] +mx = t[keep].apply(pd.to_numeric, errors='coerce').fillna(0).max(axis=1).to_dict() + +rows = [] +for parent, kids in captured['merge'].items(): + for kid in kids: + p, k = str(parent), str(kid) + rows.append((k, mz[k] - mz[p], mx[k] / mx[p] if mx[p] > 0 else np.inf, mz[k])) +d = pd.DataFrame(rows, columns=['removed', 'dm', 'ratio', 'mz']) +frac = d['dm'] - np.floor(d['dm']) +n13 = np.round(d['dm'] / 1.00335) +iso = (n13 >= 1) & ((d['dm'] - n13 * 1.00335).abs() < 0.01) +half = (frac - 0.5017).abs() < 0.01 +near = d['dm'] < 0.5 +d['kind'] = np.select([iso, half, near], ['13C isotope spacing (n x 1.0034 +/- 0.01)', + 'half-integer spacing (2+ ion)', + 'within 0.5 Da of parent (ringing band)'], + 'other spacing in 1.0-1.5 / 2.0-2.5 Da band') +d['md_fail'] = ~prfilter.md_plausible(d['mz'].to_numpy(), 0.00112, 0.03, -0.00025, -0.03) + +print(f'\n{len(d)} features removed') +print(f'{"kind":46} {"n":>4} {"weaker than parent":>19} {"<1% of parent":>14} {"fails mass defect":>18}') +for kind, g in d.groupby('kind'): + print(f'{kind:46} {len(g):4d} {int((g.ratio < 1).sum()):19d} {int((g.ratio < 0.01).sum()):14d} {int(g.md_fail.sum()):18d}') +print(f'{"all":46} {len(d):4d} {int((d.ratio < 1).sum()):19d} {int((d.ratio < 0.01).sum()):14d} {int(d.md_fail.sum()):18d}') +print('removed feature STRONGER than the parent it was merged into:', int((d.ratio > 1).sum())) +print('intensity ratio removed/parent, quartiles:', np.percentile(d.ratio.replace(np.inf, 1e9), [10, 25, 50, 75, 90]).round(3)) diff --git a/code/MSFaST.py b/code/MSFaST.py index 40df133..33333a9 100644 --- a/code/MSFaST.py +++ b/code/MSFaST.py @@ -193,6 +193,9 @@ def run_MSFaST(params): # The GUI currently forces grpave=True, but loaded sessions/tests need not. groupionlists = {} if analysis_params.relfil: + # 质量亏损过滤(在relfil/decon之前,先清掉化学上不可能的噪声离子) + if getattr(analysis_params, 'mdfil', False): + ionfilters = filter.mdfilter(analysis_params, ionfilters) ionfilters = filter.relationalfilter(analysis_params, ionfilters) if analysis_params.merge: filter.mergeions(analysis_params, ionfilters) @@ -212,6 +215,9 @@ def run_MSFaST(params): ionfilters = filter.cvfilter(analysis_params, ionfilters, analysis_params.cvthresh) if analysis_params.decon: ionfilters = filter.decon(analysis_params, ionfilters) + if getattr(analysis_params, 'adductgrp', False): # 新增:加合物/在源裂解/二聚体归组 + adduct_removed = filter.adductgroup(analysis_params, getattr(analysis_params, 'adduct_list', None)) + ionfilters['adduct'] = filter.ionfilter('', list(adduct_removed)) filter.applyfilters(analysis_params, ionfilters) if analysis_params.prperr: stats.properr(analysis_params) @@ -220,6 +226,7 @@ def run_MSFaST(params): groupionlists['cv'] = ionfilters['cv'].ions if analysis_params.CVfil else [] groupionlists['relfil'] = ionfilters['relfil'].ions if analysis_params.relfil else [] groupionlists['insource'] = ionfilters['insource'].ions if analysis_params.decon else [] + groupionlists['adduct'] = ionfilters['adduct'].ions if getattr(analysis_params, 'adductgrp', False) else [] # Add groups column to iondict csv iondict = pd.read_csv(analysis_params.outputdir / 'iondict.csv', sep=',', header=0, index_col=None) diff --git a/code/adduct_config.txt b/code/adduct_config.txt new file mode 100644 index 0000000..7035e02 --- /dev/null +++ b/code/adduct_config.txt @@ -0,0 +1,86 @@ +# ============================================================ +# MPACT 加合物 / 中性丢失(在源裂解)归组 配置 +# 负离子模式 | 乙腈水 + 乙酸铵 + 氨水 (碱性 HILIC) +# 放在 code 文件夹(和 main.py 同目录)。改完保存,重跑生效。 +# +# English: settings for MPACT's beta features. Keep this file next +# to main.py; edit, save and re-run. Everything is OFF by default. +# The ion forms below are an example for negative mode (acetonitrile/ +# water + ammonium acetate + ammonia, basic HILIC). Do not put a +# comment on the same line as a setting. +# ============================================================ + +# adduct / neutral-loss grouping: True or False +enable = False +# pos: mass differences are relative to [M+H]+ ; neg: relative to [M-H]- +mode = neg +# m/z tolerance for an adduct match (ppm) +ppm = 3 +# ion forms must correlate across samples: Pearson r > corr_cut, p < corr_p +corr_cut = 0.75 +corr_p = 0.05 +# mass defect filter: True or False +mass_defect = False +# widen the mass defect band for compounds with several Cl / Br +md_halogen = False +# peak correction algorithm: legacy (published) or beta +peak_correction = legacy + +# ============================================================ +# 离子形式:质量差相对 [M-H]-,单位 Da +# 加合为正,中性丢失为负。不用的行首加 # 即可 +# +# English: ion forms as name, mass difference from the base ion (Da). +# Adducts are positive, neutral losses negative. Put # in front of a +# line to switch it off. For positive mode set mode = pos, switch the +# negative forms off and use the positive block further down. +# ============================================================ + +# --- 加合物 --- +M+Hac-H, 60.0211 +M+Cl, 35.9767 + +# --- 中性丢失 / 在源裂解 --- +M-H2O-H, -18.0106 +M-2H2O-H, -36.0211 +M-CO2-H, -43.9898 +M-CO-H, -27.9949 +M-SO3-H, -79.9568 +M-H2SO4-H, -97.9674 +M-GlcA-H, -176.0321 +M-Hex-H, -162.0528 +M-Taurine-H,-125.0147 + +# --- 备用(需要时取消注释)--- +# M+Na-2H, 21.9819 +# M+K-2H, 37.9559 +# M+Cl37, 37.9737 +# M+FA-H, 46.0055 + +# --- positive mode, relative to [M+H]+ (remove the # to use) --- +# M+Na, 21.9825 +# M+K, 37.9564 +# M+NH4, 17.0265 +# M-H2O+H, -18.0106 +# M-H4O2+H, -36.0211 + +# ============================================================ +# 备注: +# - 同位素 / 振铃 / 二聚体 由 relfil 处理,不在这里配置 +# - [M+Cl]- 来自样品基质,不是流动相 +# - Cl 的质量亏损 -0.031,质量亏损下界 -0.03,刚好卡边界。跑完查 +# iondict 里 pass_mdfil=False 的离子有无成对相差 35.9767 的, +# 有就把 md_halogen 改成 True 重跑 +# - 37Cl 与 35Cl 差 1.9971,不是 1.0034 的整数倍,relfil 抓不到 +# ============================================================ +# +# English notes: +# - isotopes / ringing / dimers are handled by peak correction, not here +# - [M+Cl]- comes from the sample matrix, not the mobile phase +# - Cl has a mass defect of -0.031 and the lower edge of the mass defect +# band is -0.03, so chlorinated ions sit on the boundary. After a run, +# look in iondict for pass_mdfil = False ions in pairs 35.9767 apart; +# if there are any, set md_halogen = True and re-run +# - 37Cl and 35Cl differ by 1.9971, not a multiple of 1.0034, so peak +# correction does not catch that pair +# ============================================================ diff --git a/code/filter.py b/code/filter.py index 534edec..93b5971 100644 --- a/code/filter.py +++ b/code/filter.py @@ -7,6 +7,7 @@ import numpy as np import math import scipy.cluster.hierarchy as spc +from scipy import stats import mspwriter from tqdm import tqdm @@ -324,7 +325,11 @@ def relationalfilter(analysis_params, ionfilterlist): The function returns a dictionary of ion filters, with the 'relfil' filter containing a list of ions to remove, and a 'merge' attribute which contains the merge groups (as an ionmerge object) that specify which ions to merge. """ - + # Opt-in beta replacement, see relationalfilter_beta below. Everything after + # this check is the published filter, unchanged. + if getattr(analysis_params, 'relfil_mode', 'legacy') == 'beta': + return relationalfilter_beta(analysis_params, ionfilterlist) + print('Running relational filter') pdindex = pd.read_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv'), sep = ',', header = [2], index_col = None) pdindex.drop(pdindex.columns.difference(['Compound','m/z','Retention time (min)']), axis=1, inplace=True) @@ -368,6 +373,413 @@ def relationalfilter(analysis_params, ionfilterlist): return(ionfilterlist) +# -----Beta peak correction (opt in: relfil_mode = 'beta')----- +def relationalfilter_beta(analysis_params, ionfilterlist): + """ + 改进版关系过滤:去除同位素、振铃、二聚体三类冗余峰。 + + 相比原版的改进(每处都有注释): + - 同位素:质量差=n×1.0033548(n=1~3),按ppm容差,且要求【弱于母峰】+ 跨样品相关高 + - 振铃:强度<母峰的一定比例(默认1%)+ 质量差<一定ppm(默认800ppm)——文献标准,不再用过宽的 floor(md*2)%2 判据 + - 二聚体:质量差小数部分≈0.5004(修正原版缺abs的bug)+ 弱于母峰 + 相关高 + - 全局扫描:每个峰独立在其RT窗内寻找母峰 + + 新增可调参数(analysis_params,若无则用默认): + iso_ppm(默认3)、iso_max_n(默认3)、iso_ratio_max(默认0.7) + ring_ratio_max(默认0.01)、ring_ppm(默认800) + dimer_tol(默认0.02)、corr_cut(默认0.75)、corr_p(默认0.05) + (RTwin 沿用原参数) + + English summary: beta peak correction, opt in with analysis_params.relfil_mode = 'beta' + (the legacy filter above stays the default). A feature is removed when a lighter + co-eluting feature (within RTwin) qualifies as its parent, where + ratio = feature max intensity / parent max intensity: + isotope mass difference = n x 1.0033548 (n = 1..3) within iso_ppm, ratio <= 0.7, + and (ratio < 0.5 or correlated across samples) + 2+ ion fractional mass difference within dimer_tol of 0.5004, ratio < 0.7, correlated + ringing mass difference < ring_ppm of m/z and ratio < ring_ratio_max + "Correlated" = Pearson r > corr_cut and p < corr_p over the samples where either is + detected (skipped, and passed, with fewer than four such samples). + The legacy ringingwin / isopeakwin / dimerpeakwin / maxisowin settings are not used. + """ + print('Running relational filter (beta: isotope/ringing/dimer with intensity+correlation)') + + # ---- 参数(可调,带默认)---- + iso_ppm = getattr(analysis_params, 'iso_ppm', 3.0) + iso_max_n = getattr(analysis_params, 'iso_max_n', 3) + iso_ratio_max = getattr(analysis_params, 'iso_ratio_max', 0.70) + ring_ratio = getattr(analysis_params, 'ring_ratio_max', 0.01) + ring_ppm = getattr(analysis_params, 'ring_ppm', 800.0) + dimer_tol = getattr(analysis_params, 'dimer_tol', 0.02) + corr_cut = getattr(analysis_params, 'corr_cut', 0.75) + corr_p = getattr(analysis_params, 'corr_p', 0.05) + rt_win = analysis_params.RTwin + C13 = 1.0033548 + + # ---- 读 m/z, RT ---- + pdindex = pd.read_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv'), + sep=',', header=[2], index_col=None) + pdindex.drop(pdindex.columns.difference(['Compound', 'm/z', 'Retention time (min)']), axis=1, inplace=True) + + # ---- 读各样品强度矩阵(用于强度比较 + 跨样品相关)---- + # 内存优化:用numpy数组+id->行号映射,不给每个峰单独存字典(省内存) + _full = pd.read_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv'), + sep=',', header=[0, 1, 2], index_col=[0], low_memory=False) + _lvl0 = [str(c[0]) for c in _full.columns] + # i >= 2: columns 0 and 1 are m/z and RT, not intensities + _keep = [i for i, g in enumerate(_lvl0) if i >= 2 and ('BLANK' not in g.upper()) and (g.upper() != 'QC')] + _IDS = _full.index.to_numpy() + _INT = _full.iloc[:, _keep].apply(pd.to_numeric, errors='coerce').fillna(0.0).to_numpy(dtype=np.float32) # float32省一半内存 + # IDs used as they come: not always integers (e.g. '0.80_418.1451n' from Progenesis) + _row_of = {_IDS[k]: k for k in range(len(_IDS))} + _MAX = _INT.max(axis=1) + del _full # 释放原始DataFrame + import gc; gc.collect() + + def _get_int(cid): + k = _row_of.get(cid) + return _INT[k] if k is not None else None + def _get_max(cid): + k = _row_of.get(cid) + return float(_MAX[k]) if k is not None else 0.0 + + def _corr_ok(id_a, id_b): + """跨样品Pearson相关(CAMERA标准:r>corr_cut 且 p 0) | (b > 0) + if mask.sum() < 4: + return True # 样品太少无法检验,放行(靠质量差+RT) + try: + r, pv = stats.pearsonr(a[mask], b[mask]) + except Exception: + return True + if not np.isfinite(r): + return True + return (r > corr_cut) and (pv < corr_p) + + # ---- 全局扫描:按m/z排序,每个峰在RT窗内找母峰 ---- + arr = pdindex.sort_values(['m/z']).to_numpy() # 列: Compound, m/z, RT + comp = arr[:, 0] + mz = arr[:, 1].astype(float) + rt = arr[:, 2].astype(float) + n = len(comp) + order = np.argsort(mz) + mzs = mz[order] + + cut_ions = [] + cut_set = set() + mergegroups = {} + + # 迭代到收敛(删除会改变可用母峰),最多几轮 + changed = True + rounds = 0 + while changed and rounds < 6: + changed = False + rounds += 1 + for i in range(n): + cid = comp[i] + if cid in cut_set: + continue + m0, r0 = mz[i], rt[i] + i0 = _get_max(cid) + lo = np.searchsorted(mzs, m0 - (iso_max_n + 0.1)) + hi = np.searchsorted(mzs, m0 - 1e-9) + for oi in range(lo, hi): + j = order[oi] + pid = comp[j] + if pid in cut_set or pid == cid: + continue + dm = m0 - mz[j] + if dm <= 0: + continue + if abs(rt[j] - r0) > rt_win: + continue + pmax = _get_max(pid) + if pmax <= 0: + continue + ratio = i0 / pmax + kmd = dm - math.floor(dm) + nn = round(dm / C13) + iso_dev = abs(dm - nn * C13) / m0 * 1e6 if nn >= 1 else 1e9 + + hit = False + # 同位素:整数13C间隔 + 弱于母峰(<比例上限) + (相关高 或 强度比很低) + if (1 <= nn <= iso_max_n) and (iso_dev < iso_ppm) and (0 < ratio <= iso_ratio_max): + if _corr_ok(pid, cid) or ratio < 0.5: + hit = True + # 二聚体:质量差小数部分≈0.5004(修正abs) + 弱于母峰 + 相关 + if (not hit) and (abs(kmd - 0.5004) < dimer_tol) and (ratio < 0.7): + if _corr_ok(pid, cid): + hit = True + # 振铃:强度<母峰比例 + 质量差corr_cut 且 p corr_cut, p < corr_p) joins its group. The most abundant member of a + group is kept; the others are removed and listed in adduct_attribution.csv + (abundances are not summed). The search is greedy and order dependent: "base" + means "the ion the others were measured from", not a verified [M+H]+. + """ + print('Running adduct / in-source / dimer grouping') + + ppm = getattr(analysis_params, 'adduct_ppm', 3.0) + rt_win = getattr(analysis_params, 'adduct_rt_win', getattr(analysis_params, 'RTwin', 0.033)) + corr_cut = getattr(analysis_params, 'corr_cut', 0.75) + corr_p = getattr(analysis_params, 'corr_p', 0.05) + + H=1.007276; Na=22.989770; K=38.963707; NH4=18.033823; H2O=18.010565 + if adduct_list is None: + # 默认正离子(用户可传自己勾选的). 相对 M+H 的质量差: + adduct_list = [ + ('M+Na', Na - H), # +21.982 + ('M+K', K - H), # +37.956 + ('M+NH4', NH4 - H), # +17.027 + ('M-H2O+H', -H2O), # -18.011 (在源丢1水) + ('M-H4O2+H', -2*H2O), # -36.021 (在源丢2水) + ] + form_deltas = [(name, d) for name, d in adduct_list if abs(d) > 1e-6] + + # ---- 基准离子形式:由 adduct_config.txt 的 mode 决定(pos -> M+H,neg -> M-H)---- + _mode = str(getattr(analysis_params, 'adduct_mode', '') or '').strip().lower() + if _mode.startswith('neg'): + base_form = 'M-H' + elif _mode.startswith('pos'): + base_form = 'M+H' + else: + base_form = 'M+H' + print(' Warning: no adduct_mode set; assuming positive mode (M+H)') + # 一致性检查:mode 与离子形式表对不上就提醒(防止用错配置文件) + _npos = sum(1 for _nm, _ in form_deltas if str(_nm).strip().endswith('+H')) + _nneg = sum(1 for _nm, _ in form_deltas if str(_nm).strip().endswith('-H')) + if base_form == 'M-H' and _npos > _nneg: + print(' Warning: mode=neg, but the ion-form list looks positive (%d names end in +H); check the adduct list' % _npos) + if base_form == 'M+H' and _nneg > _npos: + print(' Warning: mode=pos, but the ion-form list looks negative (%d names end in -H); check the adduct list' % _nneg) + print(' base ion form: %s (mode=%s, %d ion forms)' % (base_form, _mode or 'unset', len(form_deltas))) + + _full = pd.read_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv'), + sep=',', header=[0,1,2], index_col=[0], low_memory=False) + mz_col = _full.columns[0]; rt_col = _full.columns[1] + mz_all = pd.to_numeric(_full[mz_col], errors='coerce').to_numpy() + rt_all = pd.to_numeric(_full[rt_col], errors='coerce').to_numpy() + ids = _full.index.to_numpy() + data_cols = list(_full.columns[2:]) + samp_cols_for_corr = [c for c in data_cols if ('BLANK' not in str(c[0]).upper()) and (str(c[0]).upper()!='QC')] + INT_all = _full[data_cols].apply(pd.to_numeric, errors='coerce').fillna(0.0).to_numpy(dtype=np.float32) + CORR_mat = _full[samp_cols_for_corr].apply(pd.to_numeric, errors='coerce').fillna(0.0).to_numpy(dtype=np.float32) + row_of = {ids[k]: k for k in range(len(ids))} + del _full + import gc; gc.collect() + + def detmean(k): + v = CORR_mat[k]; nz = v[v>0] + return float(nz.mean()) if nz.size else 0.0 + detmean_all = np.array([detmean(k) for k in range(len(ids))]) + + def corr_ok(ka, kb): + a, b = CORR_mat[ka], CORR_mat[kb] + mask = (a>0)|(b>0) + if mask.sum() < 4: return True + try: r, pv = stats.pearsonr(a[mask], b[mask]) + except Exception: return True + if not np.isfinite(r): return True + return (r > corr_cut) and (pv < corr_p) + + order = np.argsort(mz_all) + mzs_sorted = mz_all[order] + assigned = {} + groups = {} + for i in range(len(ids)): + cid = ids[i] + if cid in assigned: continue + m0, r0 = mz_all[i], rt_all[i] + if not np.isfinite(m0): continue + members = [(cid, base_form)] + for name, delta in form_deltas: + target_mz = m0 + delta + lo = np.searchsorted(mzs_sorted, target_mz*(1-ppm/1e6)-0.001) + hi = np.searchsorted(mzs_sorted, target_mz*(1+ppm/1e6)+0.001) + for oi in range(lo, hi): + j = order[oi]; jid = ids[j] + if jid == cid or jid in assigned: continue + if abs(mz_all[j]-target_mz)/target_mz*1e6 > ppm: continue + if abs(rt_all[j]-r0) > rt_win: continue + if corr_ok(i, j): + members.append((jid, name)); assigned[jid] = (cid, name); break + if len(members) > 1: + groups[cid] = members; assigned[cid] = (cid, base_form) + + removed_ids = set() + attribution_rows = [] + header_cols = ['Compound','m/z','Retention time (min)','adduct_form','is_main'] + [str(c[2]) for c in data_cols] + form_map = {} + for base, members in groups.items(): + best = max(members, key=lambda x: detmean_all[row_of[x[0]]]) + main_id = best[0] + mem_forms = dict(members) + for mid, form in members: + form_map[mid] = form + if mid != main_id: removed_ids.add(mid) + def make_row(pid, form, is_main): + k = row_of[pid] + return [pid, round(mz_all[k],4), round(rt_all[k],4), form, ('main' if is_main else ''), *list(INT_all[k])] + attribution_rows.append(make_row(main_id, mem_forms[main_id], True)) + for mid, form in members: + if mid != main_id: + attribution_rows.append(make_row(mid, form, False)) + attribution_rows.append([None]*len(header_cols)) + + pd.DataFrame(attribution_rows, columns=header_cols).to_csv( + analysis_params.outputdir / 'adduct_attribution.csv', index=False) + + iondict = pd.read_csv(analysis_params.outputdir / 'iondict.csv', sep=',', header=[0], index_col=0) + iondict['pass_adduct'] = ~iondict.index.isin(removed_ids) + iondict['adduct_form'] = iondict.index.map(lambda x: form_map.get(x,'')) + iondict.to_csv(analysis_params.outputdir / 'iondict.csv', header=True, index=True) + + print(' adduct grouping: %d groups, %d redundant ions moved to attribution table' % (len(groups), len(removed_ids))) + return removed_ids + + +# -----Mass defect filter----- +def md_plausible(mz, hi_slope, hi_int, lo_slope, lo_int): + """ + 判断 m/z 是否可能是单电荷 CHNOPS 小分子离子。 + + 允许区间(按质量线性变化的质量亏损上下界): + 上界 hi(m) = hi_slope*m + hi_int (饱和烃 / 脂类, H 最多) + 下界 lo(m) = lo_slope*m + lo_int (高度磷酸化/硫酸化, 杂原子最多) + 质量亏损 = m/z - 名义质量, 名义质量未知。只要存在某个整数 N 使 + lo <= m/z - N <= hi 即视为可能。这样不会出现 floor/round 带来的"翻转" + 问题(脂类亏损 >0.5、高分子量 >1 时依然正确)。 + 注意: 区间宽度在 m/z ~650 以上超过 1 Da, 此时所有 m/z 都会通过 —— + 这是物理事实, 不是 bug: 质量亏损只能剔除低质量端的不可能离子。 + + English summary: True if the m/z could be a singly charged CHNOPS ion, i.e. some + integer nominal mass puts its mass defect inside a band whose edges are linear in + mass. Above m/z ~690 the band is wider than 1 Da, so everything passes. + """ + mz = np.asarray(mz, dtype=float) + hi = hi_slope * mz + hi_int + lo = lo_slope * mz + lo_int + return np.ceil(mz - hi) <= np.floor(mz - lo) + + +def mdfilter(analysis_params, ionfilterlist): + """ + 质量亏损过滤:剔除 m/z 在化学上不可能对应单电荷 CHNOPS 分子的离子 + (噪声、盐簇、背景离子、未识别的多电荷离子等)。 + + 结果写入: + - ionfilterlist['massdefect'] + - iondict.csv 新增 'mass_defect'(m/z - round(m/z))与 'pass_mdfil' + (applyfilters 会自动应用所有 pass_* 列) + 若 md_drop_early=True(默认), 还会把不通过的离子直接从 _formatted.csv 删除, + 避免它们参与后续 decon(在源碎片聚类会把最高 m/z 当母离子, 一个不可能的 + 高 m/z 噪声可能把真实离子误判为它的碎片)和 relationalfilter。 + 因此请在 relationalfilter / decon 之前调用。 + + 可调参数(analysis_params, 带默认): + md_hi_slope (0.00112) md_hi_int (0.03) 上界 + md_lo_slope (-0.00025) md_lo_int (-0.03) 下界 + md_halogen (False) True 时下界放宽到含多个 Cl/Br 的水平 + md_drop_early (True) + 默认值按 [M+H]+ 估计, ±0.03 的截距余量覆盖 [M+Na]+、[M+NH4]+、[M-H]- 等 + 常见加合物的偏移。若有 HMDB/RefMet 拟合的包络线, 直接替换这四个数即可。 + + English summary: beta mass defect filter, opt in with analysis_params.mdfil = True. + Flags ions that fail md_plausible (pass_mdfil in iondict) and, with md_drop_early + (the default), deletes them from _formatted.csv so they take no part in decon or + relationalfilter. Run it before those. + """ + print('Running mass defect filter') + + hi_slope = getattr(analysis_params, 'md_hi_slope', 0.00112) + hi_int = getattr(analysis_params, 'md_hi_int', 0.03) + lo_slope = getattr(analysis_params, 'md_lo_slope', -0.00025) + lo_int = getattr(analysis_params, 'md_lo_int', -0.03) + if getattr(analysis_params, 'md_halogen', False): + lo_slope = getattr(analysis_params, 'md_lo_slope_halogen', -0.0011) + lo_int = getattr(analysis_params, 'md_lo_int_halogen', -0.05) + drop_early = getattr(analysis_params, 'md_drop_early', True) + + fpath = analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv') + idx = pd.read_csv(fpath, sep=',', header=[2], index_col=None) + idx = idx[['Compound', 'm/z']].copy() + idx['m/z'] = pd.to_numeric(idx['m/z'], errors='coerce') + + ok = md_plausible(idx['m/z'].to_numpy(), hi_slope, hi_int, lo_slope, lo_int) + ok = ok & idx['m/z'].notna().to_numpy() + fail_ids = idx.loc[~ok, 'Compound'].tolist() + fail_str = set(str(x) for x in fail_ids) + + ionfilterlist['massdefect'] = ionfilter( + {'hi_slope': hi_slope, 'hi_int': hi_int, 'lo_slope': lo_slope, 'lo_int': lo_int}, + fail_ids) + + # iondict: 写质量亏损和 pass 列(用字符串匹配 ID, 避免 int/str 类型不一致) + md_map = dict(zip(idx['Compound'].astype(str), idx['m/z'] - np.round(idx['m/z']))) + iondict = pd.read_csv(analysis_params.outputdir / 'iondict.csv', sep=',', header=[0], index_col=0) + key = iondict.index.astype(str) + iondict['mass_defect'] = [md_map.get(k, np.nan) for k in key] + iondict['pass_mdfil'] = ~key.isin(fail_str) + iondict.to_csv(analysis_params.outputdir / 'iondict.csv', header=True, index=True) + + if drop_early and fail_ids: + msdata = pd.read_csv(fpath, sep=',', header=[0, 1, 2], index_col=[0]) + msdata = msdata[~msdata.index.astype(str).isin(fail_str)] + msdata.to_csv(fpath, header=True, index=True) + + print(' mass defect filter: %d / %d ions failed' % (len(fail_ids), len(idx))) + return ionfilterlist + + def mergeions(analysis_params, ion_filters): """ Merges source to target ion abundances for mispicked peak filtering. @@ -421,17 +833,31 @@ def mergeions(analysis_params, ion_filters): def applyfilters(analysis_params, ion_filters): """ - Applies a list of ion filters to the data. - - Parameters: - analysis_params (object): Analysis parameters. - ion_filters (dict): Dictionary of ion filters. - - Returns: - None + Apply filters to the data. + + 改进版:以 iondict 的 pass_* 列为唯一依据删除,确保 filtered 表与 iondict 完全同步。 + (原版用 ion_filters 字典,在多过滤器/多轮运行时易与 iondict 的 pass 列不同步, + 导致 filtered 里残留应删除的峰。) + 只有所有 pass_* 列都为 True 的特征才保留。 + + English summary: a feature is kept only if every pass_* column in iondict is True, + so the filtered table and iondict cannot disagree. """ - ms_data_out_filtered = pd.read_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv'), sep=',', header=None, index_col=None, low_memory=False) # reopen original data - for elem in ion_filters: - ms_data_out_filtered = listfilter(ms_data_out_filtered, ion_filters[elem].ions, False) - ms_data_out_filtered.to_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_filtered.csv'), index=False, header=False) # saves output - + ms_data_out_filtered = pd.read_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_formatted.csv'), + sep=',', header=None, index_col=None, low_memory=False) + # 读 iondict,取所有 pass_* 列 + iondict = pd.read_csv(analysis_params.outputdir / 'iondict.csv', sep=',', header=[0], index_col=0) + passcols = [c for c in iondict.columns if str(c).startswith('pass')] + # 全部 pass 为 True 的 Compound 才保留 + keep_mask = pd.Series(True, index=iondict.index) + for _c in passcols: + keep_mask = keep_mask & (iondict[_c].astype(str).str.strip().str.upper() == 'TRUE') + keep_ids = set(iondict.index[keep_mask].astype(str)) + + # formatted 前3行是表头(组/样品/列名),数据从第4行;第一列是 Compound + header = ms_data_out_filtered.iloc[:3] + body = ms_data_out_filtered.iloc[3:] + body_keep = body[body.iloc[:, 0].astype(str).isin(keep_ids)] + out = pd.concat([header, body_keep], ignore_index=True) + out.to_csv(analysis_params.outputdir / (analysis_params.filename.stem + '_filtered.csv'), + index=False, header=False) diff --git a/code/main.py b/code/main.py index c087d4a..b169ea5 100644 --- a/code/main.py +++ b/code/main.py @@ -1400,6 +1400,67 @@ def enumerate_inputs(self): self.analysis_paramsgui.statstgrps = [str(self.ui.combo_expgrp.currentText()), str(self.ui.combo_ctrgrp.currentText())] self.analysis_paramsgui.cvthresh = float(self.ui.lineEdit_cvthresh.text()) + + # ============ 新增:读取加合物归组配置文件 adduct_config.txt ============ + # 该文件与main.py同目录,用记事本编辑即可。控制:是否开启加合物/中性丢失归组、离子模式、离子形式列表。 + # English: reads adduct_config.txt (next to main.py), which switches on the beta + # filters. Everything stays off, and peak correction stays on the published + # algorithm, unless that file says otherwise. + import os as _os + _cfg_path = _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), 'adduct_config.txt') + self.analysis_paramsgui.adductgrp = False + self.analysis_paramsgui.mdfil = False + self.analysis_paramsgui.md_halogen = False + self.analysis_paramsgui.relfil_mode = 'legacy' + self.analysis_paramsgui.adduct_list = None + self.analysis_paramsgui.adduct_mode = 'pos' + self.analysis_paramsgui.adduct_ppm = 3.0 + self.analysis_paramsgui.corr_cut = 0.75 + self.analysis_paramsgui.corr_p = 0.05 + if _os.path.exists(_cfg_path): + _adducts = [] + try: + with open(_cfg_path, 'r', encoding='utf-8') as _f: + for _line in _f: + _line = _line.strip() + if (not _line) or _line.startswith('#'): + continue + if '=' in _line and ',' not in _line: + _k, _v = [x.strip() for x in _line.split('=', 1)] + if _k.lower() == 'enable': + self.analysis_paramsgui.adductgrp = (_v.lower() in ('true', '1', 'yes', 'on')) + elif _k.lower() == 'ppm': + self.analysis_paramsgui.adduct_ppm = float(_v) + elif _k.lower() == 'corr_cut': + self.analysis_paramsgui.corr_cut = float(_v) + elif _k.lower() == 'corr_p': + self.analysis_paramsgui.corr_p = float(_v) + elif _k.lower() == 'mass_defect': + self.analysis_paramsgui.mdfil = (_v.lower() in ('true','1','yes','on')) + elif _k.lower() == 'md_halogen': + self.analysis_paramsgui.md_halogen = (_v.lower() in ('true','1','yes','on')) + elif _k.lower() == 'mode': + # 离子模式:决定基准离子形式([M+H]+ / [M-H]-) + self.analysis_paramsgui.adduct_mode = _v.strip().lower() + elif _k.lower() == 'peak_correction': + self.analysis_paramsgui.relfil_mode = 'beta' if _v.strip().lower() == 'beta' else 'legacy' + elif ',' in _line: + _parts = [x.strip() for x in _line.split(',')] + if len(_parts) >= 2: + try: + _adducts.append((_parts[0], float(_parts[1]))) + except ValueError: + pass + if _adducts: + self.analysis_paramsgui.adduct_list = _adducts + print('adduct_config.txt loaded: mode=%s, adduct=%s, mass_defect=%s, peak_correction=%s, %d ion forms, ppm=%s' % + (self.analysis_paramsgui.adduct_mode, self.analysis_paramsgui.adductgrp, self.analysis_paramsgui.mdfil, + self.analysis_paramsgui.relfil_mode, len(_adducts), self.analysis_paramsgui.adduct_ppm)) + except Exception as _e: + print('Warning: failed to read adduct_config.txt:', _e) + else: + print('adduct_config.txt not found; beta features disabled.') + # ============ 新增结束 ============ if self.ui.radioButton_meancv.isChecked(): self.analysis_paramsgui.cvparam = 'average CV' else: diff --git a/code/tests/test_beta_filters.py b/code/tests/test_beta_filters.py new file mode 100644 index 0000000..affe1b2 --- /dev/null +++ b/code/tests/test_beta_filters.py @@ -0,0 +1,164 @@ +"""Tests for the opt-in beta filters in ``filter.py``: the mass defect filter, +adduct / neutral-loss grouping, and the beta peak-correction mode. + +The small cases build the two intermediate files those functions read +(``_formatted.csv`` and ``iondict.csv``) by hand, so each rule can be +checked on a table where the right answer is known. The last test switches all +three on for a real run of the bundled PTY087I2 example. +""" + +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +import filter +from MSFaST import run_MSFaST +from test_msfast_pipeline import make_params + +RISING = [100.0, 200.0, 300.0, 400.0, 500.0, 600.0] +SHUFFLED = [600.0, 100.0, 500.0, 200.0, 400.0, 300.0] # uncorrelated with RISING +NA_DELTA = 22.989770 - 1.007276 +K_DELTA = 38.963707 - 1.007276 +C13 = 1.0033548 + + +def write_tables(tmp_path, features): + """features: [(id, m/z, RT, [six intensities]), ...] -> params for the filters.""" + groups = ['A', 'A', 'A', 'B', 'B', 'B'] + columns = pd.MultiIndex.from_tuples( + [(f'Unnamed: {i}_level_0', f'Unnamed: {i}_level_1', name) + for i, name in enumerate(['Compound', 'm/z', 'Retention time (min)'])] + + [(group, f'sample{i}', f'injection{i}') for i, group in enumerate(groups)]) + rows = [[fid, mz, rt, *intensities] for fid, mz, rt, intensities in features] + pd.DataFrame(rows, columns=columns).to_csv(tmp_path / 'x_formatted.csv', index=False) + pd.DataFrame({'Compound': [f[0] for f in features], + 'm/z': [f[1] for f in features], + 'Retention time (min)': [f[2] for f in features]}).to_csv(tmp_path / 'iondict.csv', index=False) + return SimpleNamespace(outputdir=tmp_path, filename=Path('x.csv'), RTwin=0.02, + maxisowin=3, ringingwin=0.5, isopeakwin=0.01, dimerpeakwin=0.01) + + +def scaled(values, factor): + return [v * factor for v in values] + + +# ---- mass defect --------------------------------------------------------- + +def test_md_plausible_accepts_ordinary_ions_and_rejects_half_integer_ones(): + ok = filter.md_plausible([419.1521, 210.0803, 165.5441, 281.5609], + 0.00112, 0.03, -0.00025, -0.03) + assert ok.tolist() == [True, True, False, False] + + +def test_md_plausible_cannot_reject_anything_at_high_mass(): + mz = np.linspace(700, 2000, 500) + assert filter.md_plausible(mz, 0.00112, 0.03, -0.00025, -0.03).all() + + +def test_mdfilter_flags_and_drops_implausible_ions(tmp_path): + params = write_tables(tmp_path, [('a', 419.1521, 1.0, RISING), + ('b', 165.5441, 2.0, RISING), + ('c', 210.0803, 3.0, RISING)]) + filters = filter.mdfilter(params, {}) + + assert filters['massdefect'].ions == ['b'] + iondict = pd.read_csv(tmp_path / 'iondict.csv', index_col=0) + assert iondict['pass_mdfil'].to_dict() == {'a': True, 'b': False, 'c': True} + # dropped from the working table, which must still have its three header rows + table = pd.read_csv(tmp_path / 'x_formatted.csv', header=[0, 1, 2], index_col=0) + assert table.index.tolist() == ['a', 'c'] + assert table.columns[0][2] == 'm/z' + + +# ---- adduct grouping ----------------------------------------------------- + +@pytest.mark.parametrize('ids', [('base', 'sodium', 'elsewhere', 'unrelated'), (1, 2, 3, 4)], + ids=['text ids', 'integer ids']) +def test_adductgroup_needs_mass_rt_and_correlation(tmp_path, ids): + base, sodium, elsewhere, unrelated = ids + params = write_tables(tmp_path, [ + (base, 300.1000, 5.00, RISING), + (sodium, 300.1000 + NA_DELTA, 5.00, scaled(RISING, 0.3)), # real [M+Na]+ + (elsewhere, 300.1000 + NA_DELTA, 7.00, scaled(RISING, 0.3)), # right mass, wrong RT + (unrelated, 300.1000 + K_DELTA, 5.00, SHUFFLED), # right mass and RT, uncorrelated + ]) + params.adduct_mode = 'pos' + + removed = filter.adductgroup(params) + + assert removed == {sodium} + iondict = pd.read_csv(tmp_path / 'iondict.csv', index_col=0) + assert iondict['pass_adduct'].tolist() == [True, False, True, True] + assert iondict.loc[base, 'adduct_form'] == 'M+H' + assert iondict.loc[sodium, 'adduct_form'] == 'M+Na' + attribution = pd.read_csv(tmp_path / 'adduct_attribution.csv').dropna(how='all') + assert attribution['Compound'].tolist() == [base, sodium] + assert attribution['is_main'].fillna('').tolist() == ['main', ''] + + +def test_adductgroup_keeps_the_most_abundant_member(tmp_path): + params = write_tables(tmp_path, [('weak_base', 300.1000, 5.0, RISING), + ('strong_sodium', 300.1000 + NA_DELTA, 5.0, scaled(RISING, 4))]) + params.adduct_mode = 'pos' + assert filter.adductgroup(params) == {'weak_base'} + + +def test_adductgroup_negative_mode_uses_the_given_list_and_base_label(tmp_path): + params = write_tables(tmp_path, [('base', 300.1000, 5.0, RISING), + ('chloride', 300.1000 + 35.9767, 5.0, scaled(RISING, 0.2))]) + params.adduct_mode = 'neg' + assert filter.adductgroup(params, [('M+Cl', 35.9767)]) == {'chloride'} + iondict = pd.read_csv(tmp_path / 'iondict.csv', index_col=0) + assert iondict['adduct_form'].to_dict() == {'base': 'M-H', 'chloride': 'M+Cl'} + + +# ---- peak correction: legacy stays the default ---------------------------- + +def peak_correction_tables(tmp_path): + return write_tables(tmp_path, [ + ('parent', 300.1000, 5.0, RISING), + ('isotope', 300.1000 + C13, 5.0, scaled(RISING, 0.2)), + # 0.2 Da above the parent and five times stronger: inside the legacy + # ringing band, but not something an intensity check calls an artefact + ('stronger_neighbour', 300.3000, 5.0, scaled(RISING, 5)), + ]) + + +def test_peak_correction_defaults_to_the_legacy_filter(tmp_path): + params = peak_correction_tables(tmp_path) + filters = filter.relationalfilter(params, {}) + assert sorted(filters['relfil'].ions) == ['isotope', 'stronger_neighbour'] + + +def test_beta_peak_correction_requires_the_removed_peak_to_be_weaker(tmp_path): + params = peak_correction_tables(tmp_path) + params.relfil_mode = 'beta' + filters = filter.relationalfilter(params, {}) + assert filters['relfil'].ions == ['isotope'] + assert filters['relfil'].merge['parent'].sources == ['isotope'] + iondict = pd.read_csv(tmp_path / 'iondict.csv', index_col=0) + assert iondict['pass_relfil'].to_dict() == {'parent': True, 'isotope': False, 'stronger_neighbour': True} + + +# ---- all three on, real data ---------------------------------------------- + +def test_pipeline_runs_with_every_beta_feature_on(tmp_path): + params = make_params(tmp_path) + params.mdfil = True + params.adductgrp = True + params.adduct_mode = 'pos' + params.relfil_mode = 'beta' + + result = run_MSFaST(params) + + assert {'massdefect', 'relfil', 'cv', 'insource', 'adduct'} <= set(result.ionfilters) + assert len(result.ionfilters['massdefect'].ions) > 0 + iondict = pd.read_csv(params.outputdir / 'iondict.csv', index_col=0) + assert {'pass_mdfil', 'pass_relfil', 'pass_adduct', 'adduct_form'} <= set(iondict.columns) + filtered = pd.read_csv(params.outputdir / (params.filename.stem + '_filtered.csv'), + header=[0, 1, 2], index_col=[0, 1, 2]) + survivors = set(filtered.index.get_level_values(0).astype(str)) + assert survivors and not survivors & set(map(str, result.ionfilters['massdefect'].ions)) diff --git a/code/tests/test_msfast_grpave_off.py b/code/tests/test_msfast_grpave_off.py index 806cc2b..1de6f5c 100644 --- a/code/tests/test_msfast_grpave_off.py +++ b/code/tests/test_msfast_grpave_off.py @@ -67,8 +67,8 @@ def test_run_msfast_with_grpave_off_does_not_raise(tmp_path): result = run_MSFaST(params) # used to raise NameError on groupionlists assert isinstance(result, AnalysisResult) assert isinstance(result.groupionlists, dict) - # The three filter keys are always added, each empty since every filter is off. - assert result.groupionlists == {'cv': [], 'relfil': [], 'insource': []} + # The filter keys are always added, each empty since every filter is off. + assert result.groupionlists == {'cv': [], 'relfil': [], 'insource': [], 'adduct': []} # With no filters applied, the filtered table is written and analysisinfo exists. assert (params.outputdir / (params.filename.stem + '_filtered.csv')).exists() assert (params.outputdir / 'analysisinfo.txt').exists()