From eb2d227bf7e246390bf96fec3163343c650b4fbc Mon Sep 17 00:00:00 2001 From: AOJDevStudio Date: Mon, 21 Sep 2026 12:20:30 -0500 Subject: [PATCH 1/4] fix(simplefin): move household merchants out of the public categorizer The public pattern table carried local businesses, the childcare provider, the church, a lender, and a county utility, all of which DataClassification.md classes as private. They are gone from the engine. Household patterns now live in the instance as merchant-rules.yaml, read by load_merchant_rules() and appended per category by merge_patterns() at sync time; a missing file is fine, a bad category name blocks the sync with a typed error. instance_init scaffolds the file. The public table keeps national brands and generic wording only. Tests and fixtures use placeholder memos and round amounts. The fidelity-mapping config drops the named direct-debit payees in favour of its existing generic pattern. CategoryRules.md is regenerated from the code table with the ordering lessons kept and every real example removed. Also brings uv.lock to 2.3.3 so the pre-commit pytest hook stops rewriting it. --- .../TransactionSyncing/CategoryRules.md | 643 ++++-------------- .claude/skills/TransactionSyncing/SKILL.md | 5 +- config/fidelity-mapping.json | 7 - src/cli/instance_init.py | 16 + src/config/instance_paths.py | 5 + src/integrations/simplefin/categorize.py | 134 ++-- .../simplefin/sync_expenses_db.py | 30 +- .../fixtures/fidelity_history_golden.csv | 2 +- tests/python/test_categorize.py | 72 +- tests/python/test_simplefin_expenses.py | 10 +- tests/python/test_snaptrade_db_sync.py | 4 +- uv.lock | 2 +- 12 files changed, 348 insertions(+), 582 deletions(-) diff --git a/.claude/skills/TransactionSyncing/CategoryRules.md b/.claude/skills/TransactionSyncing/CategoryRules.md index 532b30c9..7a3c3a5f 100644 --- a/.claude/skills/TransactionSyncing/CategoryRules.md +++ b/.claude/skills/TransactionSyncing/CategoryRules.md @@ -2,543 +2,202 @@ Pattern matching rules for auto-categorizing card and bank purchases. -> **Executable source of truth:** these rules are implemented in code at -> `src/integrations/simplefin/categorize.py` (`categorize_expense(text, amount)`), -> which the SimpleFIN expense sync runs so every `bank_transactions` row arrives -> pre-categorized. This document is the human-readable mirror. When you add or -> change a pattern here, update `categorize.py` (and its test) to match. - -## How to Add New Patterns - -To add a new categorization rule: - -1. Identify the merchant name pattern from Fidelity descriptions -2. Add to the appropriate category section below -3. Patterns are **case-insensitive** -4. Use partial matches (e.g., "h-e-b" matches "H-E-B #063 Pearland TX") - ---- - -## Category Patterns - -### Groceries -Supermarkets, grocery stores, food supplies - -**Patterns**: -- `h-e-b`, `heb` -- `kroger` -- `costco` -- `wal-mart`, `walmart` -- `wholefds`, `whole foods` -- `makola` -- `target` (when food context) -- `sam's club` -- `aldi` -- `trader joe` - -**Examples**: -- "H-E-B #063 Pearland TX" -> Groceries -- "COSTCO WHSE #1 PEARLAND TX" -> Groceries -- "MAKOLA IMPORTS HOUSTON TX" -> Groceries - ---- - -### Dining Out -Restaurants, fast food, entertainment dining - -**Patterns**: -- `benihana` -- `golden corral` -- `papa john` -- `chuck e cheese` -- `wingstop` -- `cinemark` -- `mcdonald` -- `chick-fil-a` -- `chipotle` -- `starbucks` -- `coffee` -- `restaurant` -- `grill` -- `cafe` -- `makiin` -- `sparkly photo` (event dining) - -**Examples**: -- "BENIHANA SUGAR LAND" -> Dining Out -- "TST*MAKIIN Houston TX" -> Dining Out -- "PAPA JOHN'S #2 PEARLAND TX" -> Dining Out - ---- - -### Auto & Transport -Vehicle expenses, fuel, parking, transportation - -**Patterns**: -- `tesla` -- `supercha` (Tesla Supercharger) -- `parking` -- `fastpark` -- `uber` -- `lyft` -- `shell` -- `exxon` -- `chevron` -- `valero` -- `buc-ee` -- `gas station` -- `toll` - -**Examples**: -- "Tesla, Inc. SUPERCHA600118984238637" -> Auto & Transport -- "FASTPARKHOU HOUSTON TX" -> Auto & Transport -- "Tesla Property Casual Fremont CA" -> Auto & Transport - ---- - -### Personal Care -Grooming, beauty, self-care - -**Patterns**: -- `salon` -- `spa` -- `barber` -- `sephora` -- `beauty supply` -- `supreme beauty` -- `ulta` -- `nail` -- `hair` -- `shaving grace` -- `gloss* skin` -- `cash app*` (often personal transfers) - -**Examples**: -- "K STAR SALON & SPA MANVEL TX" -> Personal Care -- "A SHAVING GRACE BARBER PEARLAND TX" -> Personal Care -- "SEPHORA PEACHT PEACHTREE CI GA" -> Personal Care - ---- - -### Health & Wellness -Medical, pharmacy, fitness - -**Patterns**: -- `cvs` -- `pharmacy` -- `walgreens` -- `life time` (gym) -- `doctor` -- `medical` -- `dental` -- `clinic` -- `hospital` -- `urgent care` - -**Examples**: -- "CVS/PHARMACY # MANVEL TX" -> Health & Wellness -- "LIFE TIME #320" -> Health & Wellness - ---- - -### Shopping -Retail, clothing, general merchandise - -**Patterns**: -- `marshalls` -- `target` (non-food) -- `amazon` -- `skims` -- `tj maxx` -- `ross` -- `old navy` -- `gap` -- `nordstrom` -- `macy` -- `best buy` -- `apple store` - -**Examples**: -- "MARSHALLS #877 PEARLAND TX" -> Shopping -- "SP SKIMS CHECKOUT.SKIM CA" -> Shopping - ---- - -### Family Care -Childcare, family activities, kids - -**Patterns**: -- `aqua tots` -- `brightwheel`, `brghtwhl` -- `daycare` -- `childcare` -- `school` -- `kid` -- `children` -- `pediatric` - -**Examples**: -- "AQUA TOTS - PEARLAND" -> Family Care -- "BRGHTWHL R* REDEEMER" -> Family Care - ---- - -### Bills & Utilities -Recurring bills, subscriptions, utilities - -**Patterns**: -- `autopay` -- `acctverify` -- `electric` -- `water` -- `internet` -- `comcast` -- `att` -- `verizon` -- `t-mobile` -- `netflix` -- `spotify` -- `subscription` - -**Examples**: -- "BMO ACCTVERIFY" -> Bills & Utilities - ---- - -### Cash Withdrawal -ATM and cash transactions - -**Patterns**: -- `atm` -- `cash withdrawal` -- `cash advance` - -**Examples**: -- "ATM0043 11555 MAGNOLIA PEARLAND TX" -> Cash Withdrawal -- "ATMXD10 *SEDONA LAKES MANVEL TX" -> Cash Withdrawal +> **Executable source of truth:** the rules live in code at +> `src/integrations/simplefin/categorize.py` (`categorize_expense`), which the +> SimpleFIN expense sync runs so every `bank_transactions` row arrives +> pre-categorized. This document mirrors that table and records the lessons +> behind its ordering. When you change a pattern, change the code and its test +> first, then this file. + +## Two tables: public brands and household merchants + +The public table in `categorize.py` holds **national brands and generic +wording only**. Anything that identifies a household (a local restaurant, the +daycare, the church, a lender, a county utility) is private data under +`DataClassification.md` and never goes in a tracked file. + +Household merchants live in the instance directory as **`merchant-rules.yaml`**, +a mapping of category name to a list of lowercase substrings. The sync loads it +with `load_merchant_rules()` and appends each list to its category with +`merge_patterns()`, so household patterns extend a category but never create or +reorder one. A missing file is fine; a typo in a category name blocks the sync +with a typed error rather than silently dropping rules. + +```yaml +# merchant-rules.yaml (instance directory, never committed to the engine) +Groceries: + - corner market +Family Care: + - little sprouts daycare +``` ---- +## How to add a pattern + +1. Decide whether the merchant is a national brand (public table) or a + household merchant (`merchant-rules.yaml`). When in doubt, it is household. +2. Copy the substring from the **raw bank memo**, not from SimpleFIN's cleaned + payee. Memos are truncated and abbreviated; a pattern that only matches the + payee form fails on the description alone. +3. Patterns are case-insensitive substrings. Anything shorter than about six + characters, or a common English word, needs a collision check against the + stored feed before it ships (see below). +4. For the public table, add a test in `tests/python/test_categorize.py` using + a placeholder memo and a round amount. + +## Matching algorithm + +- Amounts under $1.00 and verification memos are `Exempt` before any pattern + runs, as are SPAXX core sweeps and a `RETURNED PAYMENT FEE`. +- A retirement account returns `Retirement` for every row. +- Payroll wording, and checks on a business account, return `Payroll`. +- Then the table runs in order and **the first category with a hit wins**. +- An unmatched credit on a business account is `Business Income`. +- Everything else is `Uncategorized`. + +The sync matches against **payee and description joined**. Matching only the +description missed every payee-shaped pattern and left most debit volume +uncategorized (2026-08-04). + +## Non-spend and business categories + +Categories that move money rather than consume it are excluded from spend +totals, otherwise a card purchase is counted twice: once on the card and again +when the bill is paid. Non-spend: `Credit Card Payment`, `Crypto Deposit`, `Exempt`, `Retirement`, `Transfer`. +Business (excluded from household reviews, included in business P&L): +`Business Expense`, `Business Income`, `Payroll`. + +## Ordering rules + +Order in the table is the whole precedence mechanism. The invariants it +encodes, each of which was a live bug once: + +- `Transfer` and `Travel` come first so an explicit remittance is not read as a + merchant and "American Express Travel" is Travel, not a card payment. +- `Dining Out` precedes `Auto & Transport` so `uber eats` beats `uber`. +- `Credit Card Payment` precedes `Bills & Utilities` so a card autopay memo is + not read as a utility. **Every pattern in that category must be + payment-specific**: a bare `credit card` once booked store purchases as bill + payments, and a bare `automatic payment` would book any biller's autopay as + non-spend. +- `Fees & Interest` precedes `Entertainment`, but position alone is not + enough: an earlier category only wins if one of its patterns matches, so + every returned-payment wording is listed in fees explicitly. +- `credit card payment` is deliberately absent from `Loan Payment`. + +## Substring collisions + +Patterns are plain substrings, so a short one fires inside unrelated words. +Three that misfiled real money: `spa` matched `SPAXX` (every core sweep became +Personal Care), `credit card` matched ` Credit Card` payees, and +`uber` claimed `Uber Eats`. Fixes: `day spa` and `med spa` replace `spa`, a +SPAXX guard runs ahead of the table, and `GOOGLE WORKSPACE` (doubled space) +and `LIVING SPACES` carry explicit patterns. + +**Always replay the rules over the stored feed before and after a change.** +Removing a broad pattern is only safe once every merchant it carried has a +pattern of its own, and adding a generic word can silently reclassify history, +because the sync upserts the category on every row it touches. -### Tuition -Education expenses +## The card-payment credit leg -**Patterns**: -- `regent univer` -- `university` -- `college` -- `tuition` -- `school` -- `education` -- `coursera` -- `udemy` +A bill payment posts twice: a debit on the funding account and a credit on the +card, worded `PAYMENT - THANK YOU` or truncated by the issuer to +`AUTOMATIC PAYMENT - THANK`. `thank you` and `payment - thank` sit in Credit +Card Payment so the credit leg does not inflate income figures summed from +credits. A bounced payment posts as a `Returned Payment` debit for the amount +it unwinds; it is the mirror of a non-spend event and nets in the same category, +while the issuer's separate `RETURNED PAYMENT FEE` is real spend. -**Examples**: -- "REGENT UNIVERSPURCHASE" -> Tuition +## Deliberately left Uncategorized ---- +`Paid Check` is ambiguous by nature: a written check can be groceries, +household, or shopping. `School` is the same case (a fee and a fundraiser land +in different buckets), and so is any Apple Pay passthrough where the memo +carries only the wallet prefix and a merchant the owner has not identified. +Guessing is worse than a visible gap. Review these by hand, then add the +merchant to `merchant-rules.yaml`. -### Business Expense -Work-related purchases - -**Patterns**: -- `gumroad` -- `ups` -- `fedex` -- `office depot` -- `staples` -- `postal` -- `usps` -- `business` -- `linkedin` -- `zoom` - -**Examples**: -- "GUMROAD* SHAWN GRADY" -> Business Expense -- "POSTAL COPY CENTER-931 PEARLAND TX" -> Business Expense - ---- +## Public table -### Loan Payment -Debt payments +Generated from `CATEGORY_PATTERNS` in table order. Regenerate rather than edit +by hand. -**Patterns**: -- `wells fargo` + `draft` or `audraft` -- `loan payment` -- `mortgage` -- `car payment` -- `student loan` +### Transfer -**Examples**: -- "WELLS FARGO AUDRAFT" -> Loan Payment +`amex send`, `taptap send`, `cashed check`, `zelle`, `venmo`, `cash app`, `transferred to`, `transfer to`, `transfer withdrawal`, `transfer deposit`, `transfer from`, `instant transfer`, `webxtransfer`, `wire transfer` ---- +### Travel -### Home & Garden -Home improvement, garden, maintenance +`american express travel`, `amex travel`, `delta`, `southwest air`, `united airlines`, `american airlines`, `airline`, `enterprise`, `hertz`, `avis`, `hotel`, `airbnb`, `expedia`, `marriott`, `hilton` -**Patterns**: -- `home depot` -- `lowes` -- `sawyer` -- `smart core` -- `garden` -- `hardware` -- `furniture` +### Groceries -**Examples**: -- "SAWYER + S* SMART CORE" -> Home & Garden +`h-e-b`, `heb`, `kroger`, `costco`, `wal-mart`, `walmart`, `wholefds`, `whole foods`, `sam's club`, `aldi`, `trader joe` ---- +### Dining Out -### Crypto Deposit -Cryptocurrency deposits and transfers +`benihana`, `golden corral`, `papa john`, `chuck e cheese`, `wingstop`, `cinemark`, `mcdonald`, `chick-fil-a`, `chipotle`, `starbucks`, `coffee`, `restaurant`, `grill`, `cafe`, `uber eats`, `doordash`, `whataburger`, `panda express`, `burger king`, `auntie anne`, `thai` -**Patterns**: -- `btc deposited` -- `bitcoin` -- `fidelity crypto` -- `eth deposited` -- `crypto` +### Credit Card Payment -**Examples**: -- "0.17713256 BTC deposited" -> Crypto Deposit -- "Fidelity Crypto® 8449251033" -> Crypto Deposit +`applecard`, `gsbapayment`, `chase payment`, `amex payment`, `discover payment`, `credit card payment`, `american express credit card`, `chase credit ca`, `wf credit card`, `apple credit card`, `amex epayment`, `chase credit card`, `thank you` ---- +### Giving -### Credit Card Payment -Credit card bill payments +`church`, `tithe`, `offering`, `ministry`, `missions` -**Patterns**: -- `applecard` -- `gsbapayment` -- `chase payment` -- `amex payment` -- `discover payment` +### Auto & Transport -**Examples**: -- "DIRECT DEBIT APPLECARD GSBAPAYMENT" -> Credit Card Payment +`tesla`, `supercha`, `vehicle registration`, `vehreg`, `dmv`, `parking`, `uber`, `lyft`, `shell`, `exxon`, `chevron`, `valero`, `gas station`, `toll`, `texaco`, `bp gas`, `car wash`, `safelite`, `auto glass` ---- +### Personal Care -### Exempt -Verification transactions, zero amounts +`salon`, `day spa`, `med spa`, `massage`, `barber`, `sephora`, `beauty supply`, `ulta`, `nail`, `hair`, `lash`, `wax` -**Patterns**: -- `ifacctverify` -- `verification` -- Amount = $0.00 or < $1.00 +### Health & Wellness -**Examples**: -- "WELLS FARGO IFACCTVERIFY" -> Exempt +`cvs`, `pharmacy`, `walgreens`, `doctor`, `medical`, `dental`, `clinic`, `hospital`, `urgent care` ---- +### Shopping -## Uncategorized +`marshalls`, `amazon`, `skims`, `tj maxx`, `ross`, `old navy`, `gap`, `nordstrom`, `macy`, `best buy`, `apple store`, `fashion nova`, `burlington`, `janie & jack`, `david yurman`, `dollar general` -Any transaction not matching the above patterns is marked as **"Uncategorized"** and flagged for manual review in the sync summary. +### Family Care -**Common uncategorized reasons**: -- New merchant not in patterns -- Unusual description format -- One-time or rare purchase +`daycare`, `childcare`, `kid`, `children`, `pediatric` -**To resolve**: Add the pattern to the appropriate category above. +### Bills & Utilities ---- +`autopay`, `acctverify`, `electric`, `water`, `internet`, `comcast`, `att`, `verizon`, `t-mobile`, `netflix`, `spotify`, `subscription`, `prime video` -## Pattern Matching Algorithm +### Cash Withdrawal -```python -def categorize_expense(description: str) -> str: - desc = description.lower() +`atm`, `cash withdrawal`, `cash advance` - # Check each category's patterns - for category, patterns in CATEGORY_PATTERNS.items(): - for pattern in patterns: - if pattern in desc: - return category +### Tuition - return "Uncategorized" +`university`, `college`, `tuition`, `coursera`, `udemy` -CATEGORY_PATTERNS = { - "Groceries": ["h-e-b", "heb", "kroger", "costco", "wal-mart", "walmart", ...], - "Dining Out": ["benihana", "golden corral", "papa john", ...], - "Auto & Transport": ["tesla", "supercha", "parking", "fastpark", ...], - # ... etc -} -``` +### Business Expense ---- - -## Extending Categories - -When the user wants to add new categories or patterns: - -1. **Add to existing category**: Update the patterns list above -2. **Create new category**: Add a new section with patterns and examples -3. **Expense Tracker sync**: Ensure the category name matches Budget Planner - -**Budget Planner categories** (must match exactly): -- Groceries -- Dining Out -- Auto & Transport -- Personal Care -- Health & Wellness -- Shopping -- Family Care -- Bills & Utilities -- Cash Withdrawal -- Tuition -- Business Expense -- Loan Payment -- Home & Garden -- Crypto Deposit -- Credit Card Payment -- Exempt -- Software & Tech -- Cell Phone -- Gas -- Water -- Light Bill -- Mortgage - ---- - -**Last Updated**: 2026-08-31 -**Maintainer**: Finance Guru TransactionSyncing skill - ---- - -## Ordering rules (2026-08-04) - -`CATEGORY_PATTERNS` is an ordered dict and **the first match wins**, so placement -is behavior, not style. Three orderings are load-bearing: - -1. **Travel before Credit Card Payment** so "American Express Travel" is a trip, - not a card payment. -2. **Credit Card Payment before Bills & Utilities.** A card autopay memo reads - "Chase Credit Cautopay", which contains the substring `autopay`. With Bills & - Utilities first, every card payment was mis-filed as a utility bill. -3. **`credit card payment` belongs to Credit Card Payment, not Loan Payment.** - It used to sit in Loan Payment, which is matched earlier, so card payments - never reached their own category. - -## Match against payee AND description - -The sync joins both fields before categorizing. `description` is the raw bank -memo (`DIRECT DEBIT AMEX EPAYMENT ACH PMT`); `payee` is SimpleFIN's normalized -merchant name (`American Express Credit Card`). Matching description alone left -67% of debit volume Uncategorized. **When adding a pattern, cover the -abbreviated memo form too**, for example `vehreg` alongside -`vehicle registration`, and `amex epayment` alongside `amex payment`. - -## Non-spend categories - -`NON_SPEND_CATEGORIES` in `categorize.py` holds `Transfer`, -`Credit Card Payment`, `Crypto Deposit`, `Exempt`, and `Retirement`. These move money rather -than consume it and **must be excluded from spend totals**. Including them -double-counts: once when a purchase hits the card, again when the card bill is -paid. On 2026-08-04 that inflated a 30-day review roughly 3x. - -## Categories added 2026-08-04 +`gumroad`, `ups`, `fedex`, `office depot`, `staples`, `postal`, `usps`, `linkedin`, `zoom`, `openai`, `chatgpt`, `anthropic`, `claude.ai`, `cursor`, `github`, `vercel`, `greptile`, `openrouter`, `slack`, `paddle`, `ui.com`, `ubiquiti`, `newegg`, `workspace` -### Giving -Church and ministry contributions. +### Loan Payment -**Patterns**: `anglicanchurch`, `church`, `tithe`, `offering`, `ministry`, `missions` +`loan payment`, `mortgage`, `mortg`, `mtgpmt`, `car payment`, `student loan` ### Fees & Interest -Card interest and bank charges, kept out of merchant spend. - -**Patterns**: `interest charge`, `finance charge`, `annual fee`, `late fee`, `overdraft`, `service charge`, `maintenance fee` - -### Additions to existing categories - -- **Transfer**: `transferred to`, `transfer to`, `transfer withdrawal`, `webxtransfer`, `wire transfer` -- **Credit Card Payment**: `credit card payment`, `american express credit card`, `chase credit ca`, `wf credit card`, `apple credit card`, `amex epayment`, `credit card` -- **Loan Payment**: `mortg`, `mtgpmt`, `aes stdnt` (bank memos abbreviate) -- **Auto & Transport**: `vehicle registration`, `vehreg`, `dmv` -- **Business Expense**: `anthropic`, `claude.ai`, `cursor`, `github`, `vercel` - -## Substring collisions fixed 2026-09-08 - -Patterns are plain substring matches, so a short pattern can fire inside a -longer, unrelated word. Three were silently misfiling real money: - -1. **`spa` matched `SPAXX`.** Fidelity's core position is the SPAXX money - market, so every core sweep, some of them five figures, was booked as - _Personal Care_. The bare pattern is gone; `day spa`, `med spa`, and - `massage` replace it, and a `spaxx` / `core account` guard returns `Exempt` - ahead of the table. -2. **`credit card` matched a store purchase.** SimpleFIN normalizes some - merchant payees to ` Credit Card`, so a Macy's shopping trip read - as a bill payment and was dropped from spend totals as non-spend. The bare - catch-all is gone. **Every pattern in Credit Card Payment must be - payment-specific.** -3. **`uber` claimed `Uber Eats`.** Food delivery was booking as a rideshare. - `uber eats` now sits in Dining Out, which is matched before Auto & Transport. - -**When adding a pattern shorter than about six characters, check it against a -real merchant list first.** Replaying the rules over the stored feed found -`spa` had also been capturing `GOOGLE WORKSPACE UNIF` (note the doubled space, -which is why the pattern is the bare word `workspace`) and `LIVING SPACES -MOBILE`. Removing a broad pattern is only safe once every merchant it had been -carrying has an explicit pattern of its own, so **always replay before and after -a rule change** rather than trusting the new patterns alone. -## The card-payment credit leg +`interest charge`, `finance charge`, `annual fee`, `annual membership fee`, `membership fee`, `late fee`, `overdraft`, `service charge`, `maintenance fee`, `maintenance charge`, `sie fee`, `adj redist`, `bounced check`, `returned check`, `nsf fee` -A bill payment posts twice: a debit on the funding account and a credit on the -card itself, worded `PAYMENT - THANK YOU` or `Payment Thank You`. Only the debit -was recognized, so the credit leg sat in Uncategorized and inflated any income -figure summed from credits. `thank you` now belongs to Credit Card Payment, -which is matched before Bills & Utilities and so also reclaims the -`AUTOPAY PAYMENT - THANK YOU` rows that `autopay` had taken. +### Entertainment -## Categories added 2026-09-08 +`playstation`, `bounce`, `gamestop`, `amc theat`, `ticketmaster` -### Entertainment -Recreation, kept separate from Dining Out so restaurants stay readable. - -**Patterns**: `playstation`, `andretti`, `bounce`, `gamestop`, `amc theat`, `ticketmaster` - -**Placed after Fees & Interest on purpose**, though position alone is not the -whole defense. An earlier category only wins if one of its patterns actually -matches, and `BOUNCED CHECK FEE` matched nothing in Fees & Interest while -containing `bounce`. Every returned-payment wording is therefore listed in Fees -& Interest explicitly. Caught by CodeRabbit on PR #176. - -### Business Income -Consulting revenue. **It has no pattern list.** The payer's name is private data -and stays out of this repository, so the account carries the signal: an -unmatched credit on an account matching `is_business_account()` is income. The -fallback runs _after_ the pattern table, so an explicit `Transfer` pattern still -claims an inter-account move first. Added to `BUSINESS_CATEGORIES`, so household -reviews exclude it and business P&L includes it. - -### Additions to existing categories - -- **Transfer**: `transfer deposit`, `transfer from`, `instant transfer`, `capital one bank` -- **Groceries**: `halal meat`, `southwest farmers` (before Dining Out, so the butcher does not match `halal guys`) -- **Dining Out**: `uber eats`, `doordash`, `whataburger`, `panda express`, `burger king`, `auntie anne`, `pick up stix`, `chicken salad chick`, `piada`, `nishiki`, `gringos`, `rouxpour`, `halal guys`, `broken egg`, `thai` -- **Auto & Transport**: `texaco`, `bp gas`, `car wash`, `parkify`, `safelite`, `auto glass` -- **Personal Care**: `clean skin`, `cloud 9 spa`, `lash`, `wax`, `face reality` -- **Health & Wellness**: `mychart`, `methodist` (Giving matches `church` first, so a congregation still lands in Giving) -- **Shopping**: `shopwss`, `fashion nova`, `burlington`, `uptown cheapskate`, `janie & jack`, `david yurman`, `dollar general` -- **Bills & Utilities**: `rhythm ops`, `total wireless`, `tidal`, `prime video` -- **Business Expense**: `greptile`, `openrouter`, `slack`, `paddle`, `ui.com`, `ubiquiti`, `newegg`, `bambula`, `connectech`, `pga frisco`, `workspace` -- **Fees & Interest**: `adj redist`, `bounced check`, `returned check`, `nsf fee` -- **Home & Garden**: `bermuda dude`, `living spaces`, `lawn`, `wayfair`, `flower shop` - -Homelab and fabrication hardware is business input, and the PGA Frisco charge is -a business trip. Both classified by the account owner 2026-09-08. +### Home & Garden -## Deliberately left Uncategorized +`home depot`, `lowes`, `garden`, `hardware`, `furniture`, `living spaces`, `lawn`, `wayfair` -`Paid Check` and `Target` are ambiguous by nature: a written check or a Target -run can be groceries, household, or shopping. Guessing is worse than a visible -gap. Review these by hand. +### Crypto Deposit -`School` is the same case, and so is any Apple Pay passthrough where the memo -carries only the wallet prefix and a merchant the owner has not identified. +`btc deposited`, `bitcoin`, `fidelity crypto`, `eth deposited`, `crypto` diff --git a/.claude/skills/TransactionSyncing/SKILL.md b/.claude/skills/TransactionSyncing/SKILL.md index 3401ef9f..3544c7ef 100644 --- a/.claude/skills/TransactionSyncing/SKILL.md +++ b/.claude/skills/TransactionSyncing/SKILL.md @@ -199,7 +199,10 @@ SnapTrade activities SimpleFIN dump (bun run src/dump.ts) ### Smart Categorization Categorization is executable and runs inside the expense adapter, so the -`category` column arrives pre-filled on every `bank_transactions` row. The rules +`category` column arrives pre-filled on every `bank_transactions` row. +Household merchants (local businesses, the daycare, the church) are private +and live in the instance as `merchant-rules.yaml`, merged into the public table +at sync time; the public table holds national brands only. The rules live in code at `src/integrations/simplefin/categorize.py` (`categorize_expense(text, amount)`), which is the source of truth mirroring the human-readable `CategoryRules.md`. Keep the two in sync when adding patterns. diff --git a/config/fidelity-mapping.json b/config/fidelity-mapping.json index e769c15a..59aaf534 100644 --- a/config/fidelity-mapping.json +++ b/config/fidelity-mapping.json @@ -279,13 +279,6 @@ }, "DIRECT_DEBIT_BILL": { "patterns": [ - "DIRECT DEBIT AES STDNT LOAN", - "DIRECT DEBIT REGENT UNIVERS", - "DIRECT DEBIT BMOBNK", - "DIRECT DEBIT CPENERGY", - "DIRECT DEBIT BRAZORIA", - "DIRECT DEBIT SAFEPOINT", - "DIRECT DEBIT CLICKPAY", "DIRECT DEBIT" ], "sign": "negative", diff --git a/src/cli/instance_init.py b/src/cli/instance_init.py index 03c3cf82..ff4e80d9 100644 --- a/src/cli/instance_init.py +++ b/src/cli/instance_init.py @@ -31,6 +31,21 @@ __pycache__/ """ +MERCHANT_RULES = """# Household merchant patterns for expense categorization. +# +# The engine's public table only knows national brands. Your own merchants (the +# local dry cleaner, the daycare, the church) go here so they never leave this +# directory. Each key is a category from src/integrations/simplefin/categorize.py +# and each value is a list of lowercase substrings matched against the joined +# payee and bank memo. Within a category, order does not matter; across +# categories, the engine's table order decides ties. +# +# Groceries: +# - corner market +# Family Care: +# - little sprouts daycare +""" + USER_PROFILE = """# Add instance-specific values after creating the instance. system_ownership: {} orientation_status: {} @@ -268,6 +283,7 @@ def _build_plan( ), PlanStep(paths.env_file, _write_text(env_content)), PlanStep(paths.profile, _write_text(USER_PROFILE)), + PlanStep(paths.merchant_rules, _write_text(MERCHANT_RULES)), ) ) if not plugin_mode: diff --git a/src/config/instance_paths.py b/src/config/instance_paths.py index 03c49112..a20054f0 100644 --- a/src/config/instance_paths.py +++ b/src/config/instance_paths.py @@ -54,6 +54,11 @@ def snaptrade_accounts(self) -> Path: """Return the SnapTrade account-routing configuration path.""" return self.root / "snaptrade-accounts.yaml" + @property + def merchant_rules(self) -> Path: + """Return the household merchant-rules file for expense categorization.""" + return self.root / "merchant-rules.yaml" + @property def system_context(self) -> Path: """Return the generated system context path.""" diff --git a/src/integrations/simplefin/categorize.py b/src/integrations/simplefin/categorize.py index 7913ffb0..4d6ba93a 100644 --- a/src/integrations/simplefin/categorize.py +++ b/src/integrations/simplefin/categorize.py @@ -1,14 +1,25 @@ -"""Categorize SimpleFIN transactions using ordered merchant patterns.""" +"""Categorize SimpleFIN transactions using ordered merchant patterns. + +The table in this module holds only national brands and generic wording. A +household's own merchants (the local dry cleaner, the daycare, the church) are +private data and live in the instance directory as ``merchant-rules.yaml``, +loaded at sync time by :func:`load_merchant_rules` and appended to the public +table by :func:`merge_patterns`. +""" import os +from collections.abc import Mapping +from pathlib import Path + +import yaml # Order matters: the first matching category wins. Transfer and Travel come # first so an explicit remittance is not read as a merchant, and so # "American Express Travel" lands in Travel rather than Credit Card Payment. CATEGORY_PATTERNS: dict[str, tuple[str, ...]] = { "Transfer": ( - "taptap send", "amex send", + "taptap send", "cashed check", "zelle", "venmo", @@ -24,8 +35,6 @@ "instant transfer", "webxtransfer", "wire transfer", - # Inbound movement between the owner's own Capital One accounts. - "capital one bank", ), "Travel": ( "american express travel", @@ -53,13 +62,9 @@ "walmart", "wholefds", "whole foods", - "makola", "sam's club", "aldi", "trader joe", - # Precedes Dining Out so the butcher does not match "halal guys". - "halal meat", - "southwest farmers", ), "Dining Out": ( "benihana", @@ -76,8 +81,6 @@ "restaurant", "grill", "cafe", - "makiin", - "sparkly photo", # Must precede Auto & Transport, whose "uber" pattern would otherwise # book a food delivery as a rideshare. "uber eats", @@ -86,14 +89,6 @@ "panda express", "burger king", "auntie anne", - "pick up stix", - "chicken salad chick", - "piada", - "nishiki", - "gringos", - "rouxpour", - "halal guys", - "broken egg", "thai", ), # Must precede Bills & Utilities: a card autopay string such as @@ -121,7 +116,6 @@ # totals as non-spend. Keep every pattern here payment-specific. ), "Giving": ( - "anglicanchurch", "church", "tithe", "offering", @@ -135,20 +129,17 @@ "vehreg", "dmv", "parking", - "fastpark", "uber", "lyft", "shell", "exxon", "chevron", "valero", - "buc-ee", "gas station", "toll", "texaco", "bp gas", "car wash", - "parkify", "safelite", "auto glass", ), @@ -162,33 +153,22 @@ "barber", "sephora", "beauty supply", - "supreme beauty", "ulta", "nail", "hair", - "shaving grace", - "cash app", - "clean skin", - "cloud 9 spa", "lash", "wax", - "face reality", ), "Health & Wellness": ( "cvs", "pharmacy", "walgreens", - "life time", "doctor", "medical", "dental", "clinic", "hospital", "urgent care", - "mychart", - # Giving matches "church" first, so a Methodist congregation still - # lands in Giving and only the outpatient centers arrive here. - "methodist", ), "Shopping": ( "marshalls", @@ -202,18 +182,13 @@ "macy", "best buy", "apple store", - "shopwss", "fashion nova", "burlington", - "uptown cheapskate", "janie & jack", "david yurman", "dollar general", ), "Family Care": ( - "aqua tots", - "brightwheel", - "brghtwhl", "daycare", "childcare", "kid", @@ -233,14 +208,10 @@ "netflix", "spotify", "subscription", - "rhythm ops", - "total wireless", - "tidal", "prime video", ), "Cash Withdrawal": ("atm", "cash withdrawal", "cash advance"), "Tuition": ( - "regent univer", "university", "college", "tuition", @@ -268,14 +239,9 @@ "openrouter", "slack", "paddle", - # Homelab and fabrication hardware. Classified as business input by the - # account owner 2026-09-08, alongside the existing developer tooling. "ui.com", "ubiquiti", "newegg", - "bambula", - "connectech", - "pga frisco", # "GOOGLE WORKSPACE" contains "spa" and was landing in Personal Care. # Matched on the bare word: the bank memo doubles the space after GOOGLE. "workspace", @@ -286,11 +252,10 @@ "Loan Payment": ( "loan payment", "mortgage", - "mortg", # Fidelity/Truist abbreviate: "TRUIST MORTG TEL MTGPMT" + "mortg", # bank memos abbreviate: "ANYBANK MORTG TEL MTGPMT" "mtgpmt", "car payment", "student loan", - "aes stdnt", ), "Fees & Interest": ( "interest charge", @@ -317,7 +282,6 @@ # returned-payment wording is listed there explicitly. "Entertainment": ( "playstation", - "andretti", "bounce", "gamestop", "amc theat", @@ -326,17 +290,13 @@ "Home & Garden": ( "home depot", "lowes", - "sawyer", - "smart core", "garden", "hardware", "furniture", - "bermuda dude", # "LIVING SPACES" contains "spa" and was landing in Personal Care. "living spaces", "lawn", "wayfair", - "flower shop", ), "Crypto Deposit": ( "btc deposited", @@ -347,6 +307,63 @@ ), } +PatternTable = dict[str, tuple[str, ...]] + + +class MerchantRulesError(ValueError): + """The instance merchant-rules file is malformed.""" + + +def load_merchant_rules(path: Path) -> PatternTable: + """Read household merchant patterns from the instance. + + The file maps a category name to a list of lowercase substrings, for + example ``Family Care: [little sprouts daycare]``. A missing file means + the household has no private rules yet and is not an error. + + Raises: + MerchantRulesError: On a category the public table does not define or + a value that is not a list of strings, so a typo blocks the sync + instead of silently dropping rules. + """ + if not path.is_file(): + return {} + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(loaded, dict): + raise MerchantRulesError(f"{path} must be a mapping of category to patterns") + rules: PatternTable = {} + for category, patterns in loaded.items(): + if category not in CATEGORY_PATTERNS: + raise MerchantRulesError( + f"{path}: unknown category {category!r}; " + f"choose one of {', '.join(CATEGORY_PATTERNS)}" + ) + if patterns is None: + continue + if not isinstance(patterns, list) or not all( + isinstance(item, str) and item.strip() for item in patterns + ): + raise MerchantRulesError( + f"{path}: {category!r} must be a list of non-empty strings" + ) + rules[category] = tuple(item.strip().lower() for item in patterns) + return rules + + +def merge_patterns( + base: Mapping[str, tuple[str, ...]], extra: Mapping[str, tuple[str, ...]] +) -> PatternTable: + """Append household patterns to each category of the public table. + + Category order, and so match precedence, comes from ``base``. Household + patterns only extend a category; they never create or reorder one. + """ + return { + category: patterns + tuple(extra.get(category, ())) + for category, patterns in base.items() + } + + # Categories that move money rather than consume it. Expense reviews should # exclude these from spend totals, otherwise a card purchase is counted twice: # once on the card account and again when the card bill is paid. @@ -426,6 +443,7 @@ def categorize_expense( text: str | None, amount: float | None = None, account_name: str | None = None, + patterns: Mapping[str, tuple[str, ...]] | None = None, ) -> str: """Return the first matching expense category. @@ -435,6 +453,9 @@ def categorize_expense( account_name: Owning account name. Required to distinguish business payroll from personal spending, because identical memo text ("Paid Check") means different things by account. + patterns: Pattern table to match against. Defaults to the public + table; the sync passes the public table merged with the + household's ``merchant-rules.yaml``. Returns: The matching category, ``Exempt``, or ``Uncategorized``. @@ -467,8 +488,9 @@ def categorize_expense( ): return "Payroll" - for category, patterns in CATEGORY_PATTERNS.items(): - if any(pattern in normalized for pattern in patterns): + table = CATEGORY_PATTERNS if patterns is None else patterns + for category, category_patterns in table.items(): + if any(pattern in normalized for pattern in category_patterns): return category # Unmatched money arriving in a business account is revenue. Running this diff --git a/src/integrations/simplefin/sync_expenses_db.py b/src/integrations/simplefin/sync_expenses_db.py index a0115446..c95d2e65 100644 --- a/src/integrations/simplefin/sync_expenses_db.py +++ b/src/integrations/simplefin/sync_expenses_db.py @@ -13,7 +13,13 @@ from typing import Any from src.config.instance_paths import InstancePaths, _db_path, load_instance_env -from src.integrations.simplefin.categorize import categorize_expense +from src.integrations.simplefin.categorize import ( + CATEGORY_PATTERNS, + PatternTable, + categorize_expense, + load_merchant_rules, + merge_patterns, +) class SimpleFinSyncError(RuntimeError): @@ -160,7 +166,10 @@ def resolve_direction(text: str | None, amount: float | None) -> str: def normalize_transaction( - account: dict[str, Any], txn: dict[str, Any], now: str + account: dict[str, Any], + txn: dict[str, Any], + now: str, + patterns: PatternTable | None = None, ) -> tuple[Any, ...]: """Normalize a SimpleFIN transaction into database column order. @@ -168,6 +177,8 @@ def normalize_transaction( account: SimpleFIN account containing the transaction. txn: SimpleFIN transaction payload. now: ISO timestamp for the current sync. + patterns: Category table, the public table merged with the household + rules. Defaults to the public table alone. Returns: A tuple matching the ``bank_transactions`` column order. @@ -202,7 +213,7 @@ def normalize_transaction( txn.get("description"), amount, direction, - categorize_expense(category_text, amount, account.get("name")), + categorize_expense(category_text, amount, account.get("name"), patterns), now, ) @@ -213,6 +224,7 @@ def sync( months: int = 12, app_dir: Path = DEFAULT_APP_DIR, dump_provider: Callable[[int, Path], dict[str, Any]] = run_dump, + merchant_rules: Path | None = None, ) -> dict[str, Any]: """Sync SimpleFIN transactions into the shared SQLite database. @@ -221,6 +233,8 @@ def sync( months: Number of months of history to request. app_dir: Directory containing the SimpleFIN Bun application. dump_provider: Callable that returns a SimpleFIN account-set payload. + merchant_rules: Household ``merchant-rules.yaml``; ``None`` categorizes + with the public table alone. Returns: Counts and metadata describing the completed sync. @@ -229,6 +243,10 @@ def sync( SimpleFinSyncError: If the provider returns an invalid payload. """ db = _db_path(database_url) + patterns = merge_patterns( + CATEGORY_PATTERNS, + load_merchant_rules(merchant_rules) if merchant_rules else {}, + ) now = datetime.now(UTC).replace(microsecond=0).isoformat() payload = dump_provider(months, app_dir) if not isinstance(payload, dict) or "accounts" not in payload: @@ -257,7 +275,7 @@ def sync( if not txn.get("id"): skipped_missing_id += 1 continue - row = normalize_transaction(account, txn, now) + row = normalize_transaction(account, txn, now, patterns) rows.append(row) category = str(row[10]) by_category[category] = by_category.get(category, 0) + 1 @@ -345,7 +363,9 @@ def main(argv: list[str] | None = None) -> int: return 0 try: - summary = sync(database_url, months=args.months) + summary = sync( + database_url, months=args.months, merchant_rules=paths.merchant_rules + ) except (SimpleFinSyncError, ValueError, RuntimeError, OSError) as exc: print(f"Sync error: {exc}", file=sys.stderr) return 1 diff --git a/tests/python/fixtures/fidelity_history_golden.csv b/tests/python/fixtures/fidelity_history_golden.csv index e8efffb7..989317e6 100644 --- a/tests/python/fixtures/fidelity_history_golden.csv +++ b/tests/python/fixtures/fidelity_history_golden.csv @@ -7,4 +7,4 @@ Run Date,Action,Symbol,Description,Type,Price ($),Quantity,Commission ($),Fees ( 05/02/2026,"REINVESTMENT as of Apr-30-2026 J P MORGAN EXCHANGE TRADED FD EQUITY... (JEPI) (Cash)",JEPI,"J P MORGAN EXCHANGE TRADED FD EQUITY",Cash,55.00,1.818,,,,-100.00,-4150.00, 05/01/2026,"JOURNALED JNL VS A/C TYPES as of May-01-2026 (Cash)",,"No Description",Cash,,0.000,,,,200.00,-4050.00, 04/30/2026,"DIRECT DEBIT APPLECARD GSBAPAYMENT (Cash)", ,"No Description",Cash,,0.000,,,,-1500.00,-4250.00, -04/29/2026,"DEBIT CARD PURCHASE H-E-B #458 PEARLAND TX 042926 AUTHID:123456 (Cash)",,"No Description",Cash,,0.000,,,,-75.00,-2750.00, +04/29/2026,"DEBIT CARD PURCHASE H-E-B #458 ANYTOWN TX 042926 AUTHID:123456 (Cash)",,"No Description",Cash,,0.000,,,,-75.00,-2750.00, diff --git a/tests/python/test_categorize.py b/tests/python/test_categorize.py index dd8ad4c7..ee6148cd 100644 --- a/tests/python/test_categorize.py +++ b/tests/python/test_categorize.py @@ -9,9 +9,13 @@ from src.integrations.simplefin.categorize import ( BUSINESS_CATEGORIES, + CATEGORY_PATTERNS, NON_SPEND_CATEGORIES, + MerchantRulesError, categorize_expense, is_business_account, + load_merchant_rules, + merge_patterns, ) @@ -157,10 +161,10 @@ def test_card_payments_are_credit_card_payment(self, text: str) -> None: assert categorize_expense(text, -9402.47) == "Credit Card Payment" def test_real_loans_still_route_to_loan_payment(self) -> None: - assert categorize_expense("DIRECT DEBIT AES STDNT LOAN", -4169.25) == ( + assert categorize_expense("DIRECT DEBIT STUDENT LOAN SVC", -4169.25) == ( "Loan Payment" ) - assert categorize_expense("TRUIST MORTG TEL MTGPMT", -1500.00) == ( + assert categorize_expense("ANYBANK MORTG TEL MTGPMT", -1500.00) == ( "Loan Payment" ) @@ -171,7 +175,7 @@ def test_amex_travel_still_beats_card_payment(self) -> None: class TestNewCategories: def test_church_giving(self) -> None: - assert categorize_expense("Anglicanchurchofpentx", -100.00) == "Giving" + assert categorize_expense("First Church of Anytown", -100.00) == "Giving" def test_interest_charge_is_a_fee(self) -> None: assert categorize_expense("Interest Charge", -267.86) == "Fees & Interest" @@ -203,7 +207,7 @@ class TestRawBankMemos: ("Transfer", "TRANSFER WITHDRAWAL To ....2222", "Transfer"), ( "State of Texas Vehicle Registration", - "BRAZORIA VEHREG 1302ANGLETON TX", + "ANYCOUNTY VEHREG 1302ANYTOWN TX", "Auto & Transport", ), ], @@ -218,7 +222,7 @@ def test_raw_memo_alone_still_matches(self) -> None: assert categorize_expense("DIRECT DEBIT AMEX EPAYMENT ACH PMT", -9402.47) == ( "Credit Card Payment" ) - assert categorize_expense("BRAZORIA VEHREG 1302ANGLETON TX", -272.00) == ( + assert categorize_expense("ANYCOUNTY VEHREG 1302ANYTOWN TX", -272.00) == ( "Auto & Transport" ) @@ -250,7 +254,7 @@ class TestUnchangedBehaviour: ("Tesla Supercharger", "Auto & Transport"), ("CVS Pharmacy", "Health & Wellness"), ("Amazon", "Shopping"), - ("Brightwheel", "Family Care"), + ("Sunny Days Daycare", "Family Care"), ("Netflix subscription", "Bills & Utilities"), ("ATM cash withdrawal", "Cash Withdrawal"), ], @@ -287,7 +291,7 @@ def test_store_purchase_is_not_a_card_payment(self) -> None: bill payment, which excluded a real purchase from spend totals.""" assert ( categorize_expense( - "Macy's Credit Card MACYS PEARLAND TWN CTR", + "Macy's Credit Card MACYS ANYTOWN TWN CTR", -255.42, "Sapphire Preferred (3333)", ) @@ -369,11 +373,7 @@ class TestEntertainment: ("memo", "expected"), [ ("PlayStation Network", "Entertainment"), - ("CE ANDRETTIS", "Entertainment"), ("Bounce", "Entertainment"), - # The account owner classified the PGA Frisco charge as a business - # trip on 2026-09-08, so it belongs to the business bucket. - ("PGA FRISCO FRONT", "Business Expense"), ], ) def test_recreation_patterns(self, memo: str, expected: str) -> None: @@ -412,7 +412,6 @@ class TestSpaPatternFallout: "Sapphire Preferred (3333)", "Home & Garden", ), - ("CLOUD 9 SPA PEARLAND", "Rewards Card (5555)", "Personal Care"), ], ) def test_merchants_the_spa_pattern_had_captured( @@ -433,3 +432,52 @@ def test_inbound_transfer_is_not_business_income(self) -> None: ) == "Transfer" ) + + +class TestMerchantRules: + """Household merchants live in the instance, never in the public table.""" + + def test_missing_file_means_no_household_rules(self, tmp_path) -> None: + assert load_merchant_rules(tmp_path / "merchant-rules.yaml") == {} + + def test_rules_extend_a_category_without_reordering(self, tmp_path) -> None: + rules = tmp_path / "merchant-rules.yaml" + rules.write_text( + "Groceries:\n - Corner Market\nFamily Care:\n - little sprouts\n" + ) + merged = merge_patterns(CATEGORY_PATTERNS, load_merchant_rules(rules)) + assert list(merged) == list(CATEGORY_PATTERNS) + assert merged["Groceries"][-1] == "corner market" + assert ( + categorize_expense("CORNER MARKET #12", -40.00, None, merged) == "Groceries" + ) + assert ( + categorize_expense("LITTLE SPROUTS", -40.00, None, merged) == "Family Care" + ) + # The public table alone still knows nothing about the household. + assert categorize_expense("LITTLE SPROUTS", -40.00, None) == "Uncategorized" + + def test_public_table_order_still_decides_ties(self, tmp_path) -> None: + """A household pattern in a later category cannot outrank an earlier + category's match; precedence stays with the public table.""" + rules = tmp_path / "merchant-rules.yaml" + rules.write_text("Shopping:\n - church store\n") + merged = merge_patterns(CATEGORY_PATTERNS, load_merchant_rules(rules)) + assert categorize_expense("CHURCH STORE", -20.00, None, merged) == "Giving" + + @pytest.mark.parametrize( + "body", + [ + "Snacks:\n - corner market\n", + "Groceries: corner market\n", + "Groceries:\n - ''\n", + "- Groceries\n", + ], + ) + def test_malformed_rules_block_instead_of_dropping( + self, tmp_path, body: str + ) -> None: + rules = tmp_path / "merchant-rules.yaml" + rules.write_text(body) + with pytest.raises(MerchantRulesError): + load_merchant_rules(rules) diff --git a/tests/python/test_simplefin_expenses.py b/tests/python/test_simplefin_expenses.py index b0cf78ac..c1efd488 100644 --- a/tests/python/test_simplefin_expenses.py +++ b/tests/python/test_simplefin_expenses.py @@ -44,7 +44,7 @@ def test_default_app_dir_is_checkout_owned(tmp_path, monkeypatch) -> None: "id": "TXN-pending", "posted": 0, "payee": "H-E-B", - "description": "H-E-B #063 PEARLAND TX", + "description": "H-E-B #063 ANYTOWN TX", "amount": "12.34", }, ], @@ -54,7 +54,7 @@ def test_default_app_dir_is_checkout_owned(tmp_path, monkeypatch) -> None: def test_categorize_expense_patterns_and_exemptions() -> None: - assert categorize_expense("H-E-B #063 PEARLAND TX") == "Groceries" + assert categorize_expense("H-E-B #063 ANYTOWN TX") == "Groceries" assert categorize_expense("CHUCK E CHEESE 578") == "Dining Out" assert categorize_expense("Tesla SUPERCHA 123") == "Auto & Transport" assert categorize_expense("OPENAI *CHATGPT SUBSCR") == "Business Expense" @@ -83,7 +83,7 @@ def test_resolve_direction_prefers_feed_wording_over_sign() -> None: # A debit marker with an unexpectedly positive amount still reads as outflow. assert resolve_direction("DIRECT DEBIT CHASE CREDIT CAUTOPAY", 600.00) == "debit" # No marker falls back to sign. - assert resolve_direction("H-E-B #063 PEARLAND TX", -54.13) == "debit" + assert resolve_direction("H-E-B #063 ANYTOWN TX", -54.13) == "debit" assert resolve_direction("Capital One", 2000.00) == "credit" assert resolve_direction(None, None) == "credit" @@ -181,7 +181,7 @@ def test_resolve_direction_handles_fidelity_card_and_transfer_wording() -> None: # Outflows the feed reported as positive. assert ( resolve_direction( - "CASH ADVANCE *SEDONA LAKES MANVEL TX 081826 AUTHID:624330 (Cash)", 654.00 + "CASH ADVANCE *LAKESIDE ANYTOWN TX 081826 AUTHID:624330 (Cash)", 654.00 ) == "debit" ) @@ -193,7 +193,7 @@ def test_resolve_direction_handles_fidelity_card_and_transfer_wording() -> None: ) assert ( resolve_direction( - "DEBIT CARD PURCHASE CASH APP*ANGLICAN CHUR cash.app TX 082726", 20.00 + "DEBIT CARD PURCHASE CASH APP*FIRST CHURCH cash.app TX 082726", 20.00 ) == "debit" ) diff --git a/tests/python/test_snaptrade_db_sync.py b/tests/python/test_snaptrade_db_sync.py index dac22568..919f7443 100644 --- a/tests/python/test_snaptrade_db_sync.py +++ b/tests/python/test_snaptrade_db_sync.py @@ -119,8 +119,8 @@ def test_categorize_matches_and_prioritizes_transfers() -> None: """Merchant text maps to the right category; transfers win over merchant reads.""" from src.integrations.simplefin.categorize import categorize_expense as categorize - assert categorize("H-E-B #063 Pearland TX") == "Groceries" - assert categorize("CVS/PHARMACY # MANVEL TX") == "Health & Wellness" + assert categorize("H-E-B #063 Anytown TX") == "Groceries" + assert categorize("CVS/PHARMACY # ANYTOWN TX") == "Health & Wellness" assert categorize("Delta Airlines") == "Travel" # "American Express Travel" must read as Travel, not a bank/transfer assert categorize("American Express Travel") == "Travel" diff --git a/uv.lock b/uv.lock index bd3df1a4..fa25bcd6 100644 --- a/uv.lock +++ b/uv.lock @@ -739,7 +739,7 @@ wheels = [ [[package]] name = "family-office" -version = "2.3.2" +version = "2.3.3" source = { editable = "." } dependencies = [ { name = "anthropic" }, From 9c46c805ef9d3e864bc13ce088266be584bc7583 Mon Sep 17 00:00:00 2001 From: AOJDevStudio Date: Mon, 21 Sep 2026 12:39:38 -0500 Subject: [PATCH 2/4] fix(simplefin): thread merchant rules through refresh_all and finish the scrub Review of the previous commit found refresh_all, the documented entry point, never passed merchant_rules, so household rules were inert on the path users actually run; a sync-boundary test now proves the file reaches the database. YAML syntax errors surface as MerchantRulesError. Remaining cent-bearing amounts, a payer token, and a lender memo in the tests and mapping config are replaced with round placeholders. graft's local cache and **/merchant-rules.yaml are gitignored, the instance layout and PRIVACY.md list the new file, and a test pins the doc's pattern table to the code. --- .../TransactionSyncing/CategoryRules.md | 11 +++- .gitignore | 5 ++ PRIVACY.md | 1 + config/fidelity-mapping.json | 4 +- .../content/docs/reference/instance-layout.md | 1 + src/integrations/refresh_all.py | 10 +++- src/integrations/simplefin/categorize.py | 5 +- .../simplefin/sync_expenses_db.py | 1 + tests/python/test_categorize.py | 56 +++++++++++++------ tests/python/test_instance_init.py | 1 + tests/python/test_simplefin_expenses.py | 54 +++++++++++++++--- 11 files changed, 116 insertions(+), 33 deletions(-) diff --git a/.claude/skills/TransactionSyncing/CategoryRules.md b/.claude/skills/TransactionSyncing/CategoryRules.md index 7a3c3a5f..b60bc372 100644 --- a/.claude/skills/TransactionSyncing/CategoryRules.md +++ b/.claude/skills/TransactionSyncing/CategoryRules.md @@ -84,6 +84,13 @@ encodes, each of which was a live bug once: every returned-payment wording is listed in fees explicitly. - `credit card payment` is deliberately absent from `Loan Payment`. +## Household patterns cannot change precedence + +A household pattern only wins from its own category's position. A merchant +whose memo contains an earlier public pattern (a butcher called "Halal Guys" +when `halal guys` sits in Dining Out) cannot be reclaimed from the instance; +that needs a change to the public table, raised as an issue. + ## Substring collisions Patterns are plain substrings, so a short one fires inside unrelated words. @@ -119,8 +126,8 @@ merchant to `merchant-rules.yaml`. ## Public table -Generated from `CATEGORY_PATTERNS` in table order. Regenerate rather than edit -by hand. +Mirrors `CATEGORY_PATTERNS` in table order; `tests/python/test_categorize.py` +asserts the two match, so edit the code first and this list second. ### Transfer diff --git a/.gitignore b/.gitignore index 68362feb..a74c6827 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ notebooks/ # User's financial profile (any location in repo tree) fin-guru/data/user-profile.yaml **/user-profile.yaml +**/merchant-rules.yaml # Onboarding state and progress (may contain sensitive configuration) .onboarding-state.json # Setup progress tracking (user-specific) @@ -170,3 +171,7 @@ reports/ # Instance notes and meeting records (household data; live in the instance directory) /.memory/ /.dev/meeting-notes/ + +# graft's local graph cache: regenerable, and it mirrors pre-scrub source text. +/graft/ +.claude/skills/graft/ diff --git a/PRIVACY.md b/PRIVACY.md index 5781b3a7..5d5dd93b 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -36,6 +36,7 @@ Finance Guru is a **private, single-user family-office system**. This document d - Raw Fidelity CSVs (gitignored) - SSNs, account numbers, and API keys (scrubbed by `src/utils/log.py:ScrubPIIProcessor` before they reach any log) - `.env` contents +- `merchant-rules.yaml`, your household merchant names (the engine ships only national brands) - `finance-guru-desktop/` runtime state (also gitignored) ## Security checklist (before publishing the fork) diff --git a/config/fidelity-mapping.json b/config/fidelity-mapping.json index 59aaf534..7c4ce97f 100644 --- a/config/fidelity-mapping.json +++ b/config/fidelity-mapping.json @@ -271,11 +271,11 @@ "notes": "Credit-card payoff. The spending was already captured at the merchant via the CC statement. Counting these as expenses double-counts spending. Only treat as 'spending' if you are NOT also importing CC statements." }, "AUTO_LOAN_PAYMENT": { - "patterns": ["DIRECT DEBIT WELLS FARGO AUDRAFT"], + "patterns": ["AUDRAFT"], "sign": "negative", "true_spending": true, "expense_category": "Loan Payment", - "notes": "Wells Fargo 'AUDRAFT' = auto loan payment (not overdraft). Real recurring expense — keep separate from CC pass-throughs." + "notes": "'AUDRAFT' = auto loan payment (not overdraft). Real recurring expense — keep separate from CC pass-throughs." }, "DIRECT_DEBIT_BILL": { "patterns": [ diff --git a/docs-site/src/content/docs/reference/instance-layout.md b/docs-site/src/content/docs/reference/instance-layout.md index aca55573..b0307568 100644 --- a/docs-site/src/content/docs/reference/instance-layout.md +++ b/docs-site/src/content/docs/reference/instance-layout.md @@ -19,6 +19,7 @@ The root is the value of `FIN_GURU_DATA_ROOT` when that variable is set and non- | `user-profile.yaml` | User profile. | | `config.yaml` | Instance configuration. | | `snaptrade-accounts.yaml` | [SnapTrade](https://snaptrade.com/) account-routing configuration. | +| `merchant-rules.yaml` | Household merchant patterns for expense categorization, merged into the engine's public brand table at sync time. | | `system-context.md` | Generated system context. | | `family_office.db` | Default family-office [SQLite](https://www.sqlite.org/) database. | | `dividend-schedules.yaml` | Dividend schedule file. | diff --git a/src/integrations/refresh_all.py b/src/integrations/refresh_all.py index aa6fedad..bb061284 100644 --- a/src/integrations/refresh_all.py +++ b/src/integrations/refresh_all.py @@ -23,11 +23,17 @@ def refresh(database_url: str | None, *, months: int = 12) -> dict[str, Any]: Returns: Per-source statuses and the refresh timestamp. """ - account_config = InstancePaths.resolve().snaptrade_accounts + paths = InstancePaths.resolve() + account_config = paths.snaptrade_accounts operations: tuple[tuple[str, Callable[[], dict[str, Any]]], ...] = ( ("positions", lambda: sync_positions(account_config, database_url)), ("transactions", lambda: sync_transactions(account_config, database_url)), - ("expenses", lambda: sync_expenses(database_url, months=months)), + ( + "expenses", + lambda: sync_expenses( + database_url, months=months, merchant_rules=paths.merchant_rules + ), + ), ) sources: list[dict[str, Any]] = [] for name, operation in operations: diff --git a/src/integrations/simplefin/categorize.py b/src/integrations/simplefin/categorize.py index 4d6ba93a..fa96b84e 100644 --- a/src/integrations/simplefin/categorize.py +++ b/src/integrations/simplefin/categorize.py @@ -328,7 +328,10 @@ def load_merchant_rules(path: Path) -> PatternTable: """ if not path.is_file(): return {} - loaded = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + try: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + raise MerchantRulesError(f"{path} is not valid YAML: {exc}") from exc if not isinstance(loaded, dict): raise MerchantRulesError(f"{path} must be a mapping of category to patterns") rules: PatternTable = {} diff --git a/src/integrations/simplefin/sync_expenses_db.py b/src/integrations/simplefin/sync_expenses_db.py index c95d2e65..c4210e0e 100644 --- a/src/integrations/simplefin/sync_expenses_db.py +++ b/src/integrations/simplefin/sync_expenses_db.py @@ -241,6 +241,7 @@ def sync( Raises: SimpleFinSyncError: If the provider returns an invalid payload. + MerchantRulesError: If ``merchant_rules`` is malformed. """ db = _db_path(database_url) patterns = merge_patterns( diff --git a/tests/python/test_categorize.py b/tests/python/test_categorize.py index ee6148cd..1b91ddcb 100644 --- a/tests/python/test_categorize.py +++ b/tests/python/test_categorize.py @@ -5,6 +5,9 @@ credit card payments to Loan Payment. """ +import re +from pathlib import Path + import pytest from src.integrations.simplefin.categorize import ( @@ -132,7 +135,7 @@ class TestTransfers: [ "Transferred to Z Cash", "Transfer to brokerage", - "DIRECT DEBIT BMOBNK CK WEBXTRANSFER", + "DIRECT DEBIT ANYBANK CK WEBXTRANSFER", "Outgoing wire transfer", ], ) @@ -158,10 +161,10 @@ class TestCreditCardPayments: ], ) def test_card_payments_are_credit_card_payment(self, text: str) -> None: - assert categorize_expense(text, -9402.47) == "Credit Card Payment" + assert categorize_expense(text, -9400.00) == "Credit Card Payment" def test_real_loans_still_route_to_loan_payment(self) -> None: - assert categorize_expense("DIRECT DEBIT STUDENT LOAN SVC", -4169.25) == ( + assert categorize_expense("DIRECT DEBIT STUDENT LOAN SVC", -4200.00) == ( "Loan Payment" ) assert categorize_expense("ANYBANK MORTG TEL MTGPMT", -1500.00) == ( @@ -170,7 +173,7 @@ def test_real_loans_still_route_to_loan_payment(self) -> None: def test_amex_travel_still_beats_card_payment(self) -> None: """Travel is matched before Credit Card Payment on purpose.""" - assert categorize_expense("American Express Travel", -813.44) == "Travel" + assert categorize_expense("American Express Travel", -800.00) == "Travel" class TestNewCategories: @@ -178,7 +181,7 @@ def test_church_giving(self) -> None: assert categorize_expense("First Church of Anytown", -100.00) == "Giving" def test_interest_charge_is_a_fee(self) -> None: - assert categorize_expense("Interest Charge", -267.86) == "Fees & Interest" + assert categorize_expense("Interest Charge", -270.00) == "Fees & Interest" def test_vehicle_registration_is_transport(self) -> None: assert ( @@ -187,7 +190,7 @@ def test_vehicle_registration_is_transport(self) -> None: ) def test_ai_tooling_is_a_business_expense(self) -> None: - assert categorize_expense("Anthropic", -210.80) == "Business Expense" + assert categorize_expense("Anthropic", -210.00) == "Business Expense" class TestRawBankMemos: @@ -219,7 +222,7 @@ def test_joined_payee_and_description( def test_raw_memo_alone_still_matches(self) -> None: """Even without the payee, the abbreviated memo must categorize.""" - assert categorize_expense("DIRECT DEBIT AMEX EPAYMENT ACH PMT", -9402.47) == ( + assert categorize_expense("DIRECT DEBIT AMEX EPAYMENT ACH PMT", -9400.00) == ( "Credit Card Payment" ) assert categorize_expense("ANYCOUNTY VEHREG 1302ANYTOWN TX", -272.00) == ( @@ -237,8 +240,8 @@ def test_membership(self) -> None: def test_double_count_case_is_excluded(self) -> None: """An Amex purchase and the Amex bill payment must not both count.""" - purchase = categorize_expense("Amazon", -329.83) - bill = categorize_expense("American Express Credit Card", -9402.47) + purchase = categorize_expense("Amazon", -330.00) + bill = categorize_expense("American Express Credit Card", -9400.00) assert purchase not in NON_SPEND_CATEGORIES assert bill in NON_SPEND_CATEGORIES @@ -280,7 +283,7 @@ class TestSubstringCollisions: ) def test_spaxx_sweep_is_exempt_not_personal_care(self, memo: str) -> None: """ "SPAXX" contains "spa", so every core sweep was booked as a spa visit.""" - assert categorize_expense(memo, 7716.47, "Cash Management (4444)") == "Exempt" + assert categorize_expense(memo, 7700.00, "Cash Management (4444)") == "Exempt" def test_real_spa_still_reaches_personal_care(self) -> None: assert categorize_expense("SERENITY DAY SPA", -120.00, None) == "Personal Care" @@ -292,7 +295,7 @@ def test_store_purchase_is_not_a_card_payment(self) -> None: assert ( categorize_expense( "Macy's Credit Card MACYS ANYTOWN TWN CTR", - -255.42, + -255.00, "Sapphire Preferred (3333)", ) == "Shopping" @@ -300,14 +303,14 @@ def test_store_purchase_is_not_a_card_payment(self) -> None: def test_actual_card_payment_still_classified(self) -> None: assert ( - categorize_expense("Chase Credit Card CHASE CREDIT CRD", -38.19, None) + categorize_expense("Chase Credit Card CHASE CREDIT CRD", -38.00, None) == "Credit Card Payment" ) def test_food_delivery_is_dining_not_rideshare(self) -> None: """ "Uber Eats" contains "uber"; Auto & Transport was claiming it.""" assert categorize_expense("Uber Eats", -1.84, None) == "Dining Out" - assert categorize_expense("Uber", -36.94, None) == "Auto & Transport" + assert categorize_expense("Uber", -37.00, None) == "Auto & Transport" class TestCardPaymentCreditLeg: @@ -325,7 +328,7 @@ class TestCardPaymentCreditLeg: ], ) def test_thank_you_credit_is_a_card_payment(self, memo: str) -> None: - assert categorize_expense(memo, 20009.65, "Platinum Card® (3333)") == ( + assert categorize_expense(memo, 20000.00, "Platinum Card® (3333)") == ( "Credit Card Payment" ) @@ -341,12 +344,12 @@ class TestBusinessIncome: PERSONAL = "360 Checking (2222)" def test_unmatched_credit_on_business_account_is_income(self) -> None: - assert categorize_expense("CGSOPERATING", 12320.00, self.BIZ) == ( + assert categorize_expense("ACME OPERATING", 12000.00, self.BIZ) == ( "Business Income" ) def test_unmatched_credit_on_personal_account_is_not_income(self) -> None: - assert categorize_expense("SOME MERCHANT", 12320.00, self.PERSONAL) == ( + assert categorize_expense("SOME MERCHANT", 12000.00, self.PERSONAL) == ( "Uncategorized" ) @@ -377,7 +380,7 @@ class TestEntertainment: ], ) def test_recreation_patterns(self, memo: str, expected: str) -> None: - assert categorize_expense(memo, -292.24, None) == expected + assert categorize_expense(memo, -290.00, None) == expected @pytest.mark.parametrize( "memo", @@ -419,7 +422,7 @@ def test_merchants_the_spa_pattern_had_captured( ) -> None: """Removing the bare "spa" pattern must not drop these into Uncategorized: each one needs its own explicit pattern.""" - assert categorize_expense(memo, -202.75, account) == expected + assert categorize_expense(memo, -200.00, account) == expected def test_inbound_transfer_is_not_business_income(self) -> None: """ "Instant Transfer Received From ...." does not contain "transfer @@ -481,3 +484,20 @@ def test_malformed_rules_block_instead_of_dropping( rules.write_text(body) with pytest.raises(MerchantRulesError): load_merchant_rules(rules) + + +def test_category_rules_doc_mirrors_the_public_table() -> None: + """CategoryRules.md lists every public pattern per category, in table order.""" + doc = ( + Path(__file__).resolve().parents[2] + / ".claude" + / "skills" + / "TransactionSyncing" + / "CategoryRules.md" + ).read_text(encoding="utf-8") + listed = doc.split("## Public table", 1)[1] + documented: dict[str, tuple[str, ...]] = {} + for block in listed.split("### ")[1:]: + heading, _, body = block.partition("\n") + documented[heading.strip()] = tuple(re.findall(r"`([^`]+)`", body)) + assert documented == CATEGORY_PATTERNS diff --git a/tests/python/test_instance_init.py b/tests/python/test_instance_init.py index 06a46d4b..1c0572a4 100644 --- a/tests/python/test_instance_init.py +++ b/tests/python/test_instance_init.py @@ -170,6 +170,7 @@ def test_fresh_root_creates_complete_instance(tmp_path: Path) -> None: root / ".gitignore", paths.env_file, paths.profile, + paths.merchant_rules, root / "AGENTS.md", root / "CLAUDE.md", ) diff --git a/tests/python/test_simplefin_expenses.py b/tests/python/test_simplefin_expenses.py index c1efd488..af187df0 100644 --- a/tests/python/test_simplefin_expenses.py +++ b/tests/python/test_simplefin_expenses.py @@ -73,7 +73,7 @@ def test_resolve_direction_prefers_feed_wording_over_sign() -> None: == "credit" ) assert ( - resolve_direction("DIRECT DEPOSIT Ridgeline SerPAYROLL (Cash)", -2273.03) + resolve_direction("DIRECT DEPOSIT Ridgeline SerPAYROLL (Cash)", -2300.00) == "credit" ) assert ( @@ -227,9 +227,9 @@ def test_retirement_account_activity_is_not_household_spending() -> None: too generic to match on (a charitable contribution is real spending). """ retirement = "EXAMPLE EMPLOYER 401(K) RETIREMENT PLAN" - assert categorize_expense("contribution", -123.08, retirement) == "Retirement" - assert categorize_expense("contribution", -30.77, retirement) == "Retirement" - assert categorize_expense("dividend", 7.84, retirement) == "Retirement" + assert categorize_expense("contribution", -120.00, retirement) == "Retirement" + assert categorize_expense("contribution", -30.00, retirement) == "Retirement" + assert categorize_expense("dividend", 8.00, retirement) == "Retirement" # Retirement must be excluded from household spending totals. assert "Retirement" in NON_SPEND_CATEGORIES # A contribution on a normal account is still ordinary spending, not Retirement. @@ -252,20 +252,58 @@ def test_inbound_direct_deposits_are_payroll_regardless_of_employer_memo() -> No """ cma = "Cash Management (Joint WROS) (0001)" assert ( - categorize_expense("DIRECT DEPOSIT ACME STAFFINGDIR DEP (Cash)", 2207.38, cma) + categorize_expense("DIRECT DEPOSIT ACME STAFFINGDIR DEP (Cash)", 2200.00, cma) == "Payroll" ) assert ( - categorize_expense("DIRECT DEPOSIT Ridgeline SerACH (Cash)", 2273.03, cma) + categorize_expense("DIRECT DEPOSIT Ridgeline SerACH (Cash)", 2300.00, cma) == "Payroll" ) # Already-working variants must not regress. assert ( - categorize_expense("DIRECT DEPOSIT NORTHFIELD COLLPAYROLL (Cash)", 2431.51, cma) + categorize_expense("DIRECT DEPOSIT NORTHFIELD COLLPAYROLL (Cash)", 2400.00, cma) == "Payroll" ) # Sub-dollar bank verification deposits stay Exempt, not payroll. assert ( - categorize_expense("DIRECT DEPOSIT WELLS FARGO ACCTVERIFY", 0.29, cma) + categorize_expense("DIRECT DEPOSIT WELLS FARGO ACCTVERIFY", 0.25, cma) == "Exempt" ) + + +def test_sync_applies_household_merchant_rules(tmp_path) -> None: + """A merchant known only to merchant-rules.yaml reaches the database with + its household category; without the file it stays Uncategorized.""" + rules = tmp_path / "merchant-rules.yaml" + rules.write_text("Family Care:\n - little sprouts\n", encoding="utf-8") + database_url = f"sqlite:///{tmp_path / 'fo.db'}" + + def fake_dump(months: int, app_dir) -> dict: + return { + "accounts": [ + { + "id": "acct-1", + "name": "Rewards Card (5555)", + "org": {"name": "Any Bank"}, + "transactions": [ + { + "id": "t1", + "posted": 1755000000, + "amount": "-40.00", + "payee": "Little Sprouts", + "description": "LITTLE SPROUTS ANYTOWN TX", + } + ], + } + ] + } + + sync(database_url, dump_provider=fake_dump) + conn = sqlite3.connect(tmp_path / "fo.db") + assert conn.execute("SELECT category FROM bank_transactions").fetchone() == ( + "Uncategorized", + ) + sync(database_url, dump_provider=fake_dump, merchant_rules=rules) + assert conn.execute("SELECT category FROM bank_transactions").fetchone() == ( + "Family Care", + ) From e22e77fdef22f341a8775cf761808d1b324fcafd Mon Sep 17 00:00:00 2001 From: Ossie Irondi Date: Thu, 24 Sep 2026 11:18:20 -0500 Subject: [PATCH 3/4] fix(simplefin): reject a category listed twice in merchant-rules.yaml yaml.safe_load keeps only the last value for a repeated key, so a second Groceries block silently dropped the first one's patterns. Load with a SafeLoader subclass that raises on a duplicate key. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/integrations/simplefin/categorize.py | 30 ++++++++++++++++++++---- tests/python/test_categorize.py | 1 + 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/integrations/simplefin/categorize.py b/src/integrations/simplefin/categorize.py index fa96b84e..e4d5447a 100644 --- a/src/integrations/simplefin/categorize.py +++ b/src/integrations/simplefin/categorize.py @@ -314,6 +314,27 @@ class MerchantRulesError(ValueError): """The instance merchant-rules file is malformed.""" +class _UniqueKeyLoader(yaml.SafeLoader): + """SafeLoader that rejects a repeated mapping key. + + Plain ``safe_load`` keeps only the last value for a repeated key, so a + second ``Groceries:`` block would silently drop the first one's patterns. + """ + + def construct_mapping( + self, node: yaml.MappingNode, deep: bool = False + ) -> dict[object, object]: + seen: set[object] = set() + for key_node, _ in node.value: + key = self.construct_object(key_node, deep=deep) + if key in seen: + raise yaml.constructor.ConstructorError( + None, None, f"duplicate key {key!r}", key_node.start_mark + ) + seen.add(key) + return super().construct_mapping(node, deep=deep) + + def load_merchant_rules(path: Path) -> PatternTable: """Read household merchant patterns from the instance. @@ -322,14 +343,15 @@ def load_merchant_rules(path: Path) -> PatternTable: the household has no private rules yet and is not an error. Raises: - MerchantRulesError: On a category the public table does not define or - a value that is not a list of strings, so a typo blocks the sync - instead of silently dropping rules. + MerchantRulesError: On a category the public table does not define, a + category listed twice, or a value that is not a list of strings, + so a typo blocks the sync instead of silently dropping rules. """ if not path.is_file(): return {} try: - loaded = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + text = path.read_text(encoding="utf-8") + loaded = yaml.load(text, Loader=_UniqueKeyLoader) or {} except yaml.YAMLError as exc: raise MerchantRulesError(f"{path} is not valid YAML: {exc}") from exc if not isinstance(loaded, dict): diff --git a/tests/python/test_categorize.py b/tests/python/test_categorize.py index 1b91ddcb..6c27cbf4 100644 --- a/tests/python/test_categorize.py +++ b/tests/python/test_categorize.py @@ -475,6 +475,7 @@ def test_public_table_order_still_decides_ties(self, tmp_path) -> None: "Groceries: corner market\n", "Groceries:\n - ''\n", "- Groceries\n", + "Groceries:\n - corner market\nGroceries:\n - farm stand\n", ], ) def test_malformed_rules_block_instead_of_dropping( From fe51388ca37beccd3b55beee32fd36c578605fab Mon Sep 17 00:00:00 2001 From: Ossie Irondi Date: Thu, 24 Sep 2026 11:27:10 -0500 Subject: [PATCH 4/4] fix(simplefin): reject empty-list, false, and unhashable-key merchant rules A [] or false root fell through 'or {}' and silently loaded no rules, and a sequence key raised TypeError instead of MerchantRulesError. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/integrations/simplefin/categorize.py | 10 ++++++++-- tests/python/test_categorize.py | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/integrations/simplefin/categorize.py b/src/integrations/simplefin/categorize.py index e4d5447a..c1c54c56 100644 --- a/src/integrations/simplefin/categorize.py +++ b/src/integrations/simplefin/categorize.py @@ -8,7 +8,7 @@ """ import os -from collections.abc import Mapping +from collections.abc import Hashable, Mapping from pathlib import Path import yaml @@ -327,6 +327,10 @@ def construct_mapping( seen: set[object] = set() for key_node, _ in node.value: key = self.construct_object(key_node, deep=deep) + if not isinstance(key, Hashable): + raise yaml.constructor.ConstructorError( + None, None, "found unhashable key", key_node.start_mark + ) if key in seen: raise yaml.constructor.ConstructorError( None, None, f"duplicate key {key!r}", key_node.start_mark @@ -351,9 +355,11 @@ def load_merchant_rules(path: Path) -> PatternTable: return {} try: text = path.read_text(encoding="utf-8") - loaded = yaml.load(text, Loader=_UniqueKeyLoader) or {} + loaded = yaml.load(text, Loader=_UniqueKeyLoader) except yaml.YAMLError as exc: raise MerchantRulesError(f"{path} is not valid YAML: {exc}") from exc + if loaded is None: + return {} if not isinstance(loaded, dict): raise MerchantRulesError(f"{path} must be a mapping of category to patterns") rules: PatternTable = {} diff --git a/tests/python/test_categorize.py b/tests/python/test_categorize.py index 6c27cbf4..32388fb9 100644 --- a/tests/python/test_categorize.py +++ b/tests/python/test_categorize.py @@ -476,6 +476,9 @@ def test_public_table_order_still_decides_ties(self, tmp_path) -> None: "Groceries:\n - ''\n", "- Groceries\n", "Groceries:\n - corner market\nGroceries:\n - farm stand\n", + "[]\n", + "false\n", + "? [Groceries, Shopping]\n: [corner market]\n", ], ) def test_malformed_rules_block_instead_of_dropping(