diff --git a/.claude/skills/TransactionSyncing/CategoryRules.md b/.claude/skills/TransactionSyncing/CategoryRules.md index 532b30c9..b60bc372 100644 --- a/.claude/skills/TransactionSyncing/CategoryRules.md +++ b/.claude/skills/TransactionSyncing/CategoryRules.md @@ -2,543 +2,209 @@ Pattern matching rules for auto-categorizing card and bank purchases. -> **Executable source of truth:** these rules are implemented in code at -> `src/integrations/simplefin/categorize.py` (`categorize_expense(text, amount)`), -> which the SimpleFIN expense sync runs so every `bank_transactions` row arrives -> pre-categorized. This document is the human-readable mirror. When you add or -> change a pattern here, update `categorize.py` (and its test) to match. - -## How to Add New Patterns - -To add a new categorization rule: - -1. Identify the merchant name pattern from Fidelity descriptions -2. Add to the appropriate category section below -3. Patterns are **case-insensitive** -4. Use partial matches (e.g., "h-e-b" matches "H-E-B #063 Pearland TX") - ---- - -## Category Patterns - -### Groceries -Supermarkets, grocery stores, food supplies - -**Patterns**: -- `h-e-b`, `heb` -- `kroger` -- `costco` -- `wal-mart`, `walmart` -- `wholefds`, `whole foods` -- `makola` -- `target` (when food context) -- `sam's club` -- `aldi` -- `trader joe` - -**Examples**: -- "H-E-B #063 Pearland TX" -> Groceries -- "COSTCO WHSE #1 PEARLAND TX" -> Groceries -- "MAKOLA IMPORTS HOUSTON TX" -> Groceries - ---- - -### Dining Out -Restaurants, fast food, entertainment dining - -**Patterns**: -- `benihana` -- `golden corral` -- `papa john` -- `chuck e cheese` -- `wingstop` -- `cinemark` -- `mcdonald` -- `chick-fil-a` -- `chipotle` -- `starbucks` -- `coffee` -- `restaurant` -- `grill` -- `cafe` -- `makiin` -- `sparkly photo` (event dining) - -**Examples**: -- "BENIHANA SUGAR LAND" -> Dining Out -- "TST*MAKIIN Houston TX" -> Dining Out -- "PAPA JOHN'S #2 PEARLAND TX" -> Dining Out - ---- - -### Auto & Transport -Vehicle expenses, fuel, parking, transportation - -**Patterns**: -- `tesla` -- `supercha` (Tesla Supercharger) -- `parking` -- `fastpark` -- `uber` -- `lyft` -- `shell` -- `exxon` -- `chevron` -- `valero` -- `buc-ee` -- `gas station` -- `toll` - -**Examples**: -- "Tesla, Inc. SUPERCHA600118984238637" -> Auto & Transport -- "FASTPARKHOU HOUSTON TX" -> Auto & Transport -- "Tesla Property Casual Fremont CA" -> Auto & Transport - ---- - -### Personal Care -Grooming, beauty, self-care - -**Patterns**: -- `salon` -- `spa` -- `barber` -- `sephora` -- `beauty supply` -- `supreme beauty` -- `ulta` -- `nail` -- `hair` -- `shaving grace` -- `gloss* skin` -- `cash app*` (often personal transfers) - -**Examples**: -- "K STAR SALON & SPA MANVEL TX" -> Personal Care -- "A SHAVING GRACE BARBER PEARLAND TX" -> Personal Care -- "SEPHORA PEACHT PEACHTREE CI GA" -> Personal Care - ---- - -### Health & Wellness -Medical, pharmacy, fitness - -**Patterns**: -- `cvs` -- `pharmacy` -- `walgreens` -- `life time` (gym) -- `doctor` -- `medical` -- `dental` -- `clinic` -- `hospital` -- `urgent care` - -**Examples**: -- "CVS/PHARMACY # MANVEL TX" -> Health & Wellness -- "LIFE TIME #320" -> Health & Wellness - ---- - -### Shopping -Retail, clothing, general merchandise - -**Patterns**: -- `marshalls` -- `target` (non-food) -- `amazon` -- `skims` -- `tj maxx` -- `ross` -- `old navy` -- `gap` -- `nordstrom` -- `macy` -- `best buy` -- `apple store` - -**Examples**: -- "MARSHALLS #877 PEARLAND TX" -> Shopping -- "SP SKIMS CHECKOUT.SKIM CA" -> Shopping - ---- - -### Family Care -Childcare, family activities, kids - -**Patterns**: -- `aqua tots` -- `brightwheel`, `brghtwhl` -- `daycare` -- `childcare` -- `school` -- `kid` -- `children` -- `pediatric` - -**Examples**: -- "AQUA TOTS - PEARLAND" -> Family Care -- "BRGHTWHL R* REDEEMER" -> Family Care - ---- - -### Bills & Utilities -Recurring bills, subscriptions, utilities - -**Patterns**: -- `autopay` -- `acctverify` -- `electric` -- `water` -- `internet` -- `comcast` -- `att` -- `verizon` -- `t-mobile` -- `netflix` -- `spotify` -- `subscription` - -**Examples**: -- "BMO ACCTVERIFY" -> Bills & Utilities - ---- - -### Cash Withdrawal -ATM and cash transactions - -**Patterns**: -- `atm` -- `cash withdrawal` -- `cash advance` - -**Examples**: -- "ATM0043 11555 MAGNOLIA PEARLAND TX" -> Cash Withdrawal -- "ATMXD10 *SEDONA LAKES MANVEL TX" -> Cash Withdrawal +> **Executable source of truth:** the rules live in code at +> `src/integrations/simplefin/categorize.py` (`categorize_expense`), which the +> SimpleFIN expense sync runs so every `bank_transactions` row arrives +> pre-categorized. This document mirrors that table and records the lessons +> behind its ordering. When you change a pattern, change the code and its test +> first, then this file. + +## Two tables: public brands and household merchants + +The public table in `categorize.py` holds **national brands and generic +wording only**. Anything that identifies a household (a local restaurant, the +daycare, the church, a lender, a county utility) is private data under +`DataClassification.md` and never goes in a tracked file. + +Household merchants live in the instance directory as **`merchant-rules.yaml`**, +a mapping of category name to a list of lowercase substrings. The sync loads it +with `load_merchant_rules()` and appends each list to its category with +`merge_patterns()`, so household patterns extend a category but never create or +reorder one. A missing file is fine; a typo in a category name blocks the sync +with a typed error rather than silently dropping rules. + +```yaml +# merchant-rules.yaml (instance directory, never committed to the engine) +Groceries: + - corner market +Family Care: + - little sprouts daycare +``` ---- +## How to add a pattern + +1. Decide whether the merchant is a national brand (public table) or a + household merchant (`merchant-rules.yaml`). When in doubt, it is household. +2. Copy the substring from the **raw bank memo**, not from SimpleFIN's cleaned + payee. Memos are truncated and abbreviated; a pattern that only matches the + payee form fails on the description alone. +3. Patterns are case-insensitive substrings. Anything shorter than about six + characters, or a common English word, needs a collision check against the + stored feed before it ships (see below). +4. For the public table, add a test in `tests/python/test_categorize.py` using + a placeholder memo and a round amount. + +## Matching algorithm + +- Amounts under $1.00 and verification memos are `Exempt` before any pattern + runs, as are SPAXX core sweeps and a `RETURNED PAYMENT FEE`. +- A retirement account returns `Retirement` for every row. +- Payroll wording, and checks on a business account, return `Payroll`. +- Then the table runs in order and **the first category with a hit wins**. +- An unmatched credit on a business account is `Business Income`. +- Everything else is `Uncategorized`. + +The sync matches against **payee and description joined**. Matching only the +description missed every payee-shaped pattern and left most debit volume +uncategorized (2026-08-04). + +## Non-spend and business categories + +Categories that move money rather than consume it are excluded from spend +totals, otherwise a card purchase is counted twice: once on the card and again +when the bill is paid. Non-spend: `Credit Card Payment`, `Crypto Deposit`, `Exempt`, `Retirement`, `Transfer`. +Business (excluded from household reviews, included in business P&L): +`Business Expense`, `Business Income`, `Payroll`. + +## Ordering rules + +Order in the table is the whole precedence mechanism. The invariants it +encodes, each of which was a live bug once: + +- `Transfer` and `Travel` come first so an explicit remittance is not read as a + merchant and "American Express Travel" is Travel, not a card payment. +- `Dining Out` precedes `Auto & Transport` so `uber eats` beats `uber`. +- `Credit Card Payment` precedes `Bills & Utilities` so a card autopay memo is + not read as a utility. **Every pattern in that category must be + payment-specific**: a bare `credit card` once booked store purchases as bill + payments, and a bare `automatic payment` would book any biller's autopay as + non-spend. +- `Fees & Interest` precedes `Entertainment`, but position alone is not + enough: an earlier category only wins if one of its patterns matches, so + every returned-payment wording is listed in fees explicitly. +- `credit card payment` is deliberately absent from `Loan Payment`. + +## Household patterns cannot change precedence + +A household pattern only wins from its own category's position. A merchant +whose memo contains an earlier public pattern (a butcher called "Halal Guys" +when `halal guys` sits in Dining Out) cannot be reclaimed from the instance; +that needs a change to the public table, raised as an issue. + +## Substring collisions + +Patterns are plain substrings, so a short one fires inside unrelated words. +Three that misfiled real money: `spa` matched `SPAXX` (every core sweep became +Personal Care), `credit card` matched ` Credit Card` payees, and +`uber` claimed `Uber Eats`. Fixes: `day spa` and `med spa` replace `spa`, a +SPAXX guard runs ahead of the table, and `GOOGLE WORKSPACE` (doubled space) +and `LIVING SPACES` carry explicit patterns. + +**Always replay the rules over the stored feed before and after a change.** +Removing a broad pattern is only safe once every merchant it carried has a +pattern of its own, and adding a generic word can silently reclassify history, +because the sync upserts the category on every row it touches. -### Tuition -Education expenses +## The card-payment credit leg -**Patterns**: -- `regent univer` -- `university` -- `college` -- `tuition` -- `school` -- `education` -- `coursera` -- `udemy` +A bill payment posts twice: a debit on the funding account and a credit on the +card, worded `PAYMENT - THANK YOU` or truncated by the issuer to +`AUTOMATIC PAYMENT - THANK`. `thank you` and `payment - thank` sit in Credit +Card Payment so the credit leg does not inflate income figures summed from +credits. A bounced payment posts as a `Returned Payment` debit for the amount +it unwinds; it is the mirror of a non-spend event and nets in the same category, +while the issuer's separate `RETURNED PAYMENT FEE` is real spend. -**Examples**: -- "REGENT UNIVERSPURCHASE" -> Tuition +## Deliberately left Uncategorized ---- +`Paid Check` is ambiguous by nature: a written check can be groceries, +household, or shopping. `School` is the same case (a fee and a fundraiser land +in different buckets), and so is any Apple Pay passthrough where the memo +carries only the wallet prefix and a merchant the owner has not identified. +Guessing is worse than a visible gap. Review these by hand, then add the +merchant to `merchant-rules.yaml`. -### Business Expense -Work-related purchases - -**Patterns**: -- `gumroad` -- `ups` -- `fedex` -- `office depot` -- `staples` -- `postal` -- `usps` -- `business` -- `linkedin` -- `zoom` - -**Examples**: -- "GUMROAD* SHAWN GRADY" -> Business Expense -- "POSTAL COPY CENTER-931 PEARLAND TX" -> Business Expense - ---- +## Public table -### Loan Payment -Debt payments +Mirrors `CATEGORY_PATTERNS` in table order; `tests/python/test_categorize.py` +asserts the two match, so edit the code first and this list second. -**Patterns**: -- `wells fargo` + `draft` or `audraft` -- `loan payment` -- `mortgage` -- `car payment` -- `student loan` +### Transfer -**Examples**: -- "WELLS FARGO AUDRAFT" -> Loan Payment +`amex send`, `taptap send`, `cashed check`, `zelle`, `venmo`, `cash app`, `transferred to`, `transfer to`, `transfer withdrawal`, `transfer deposit`, `transfer from`, `instant transfer`, `webxtransfer`, `wire transfer` ---- +### Travel -### Home & Garden -Home improvement, garden, maintenance +`american express travel`, `amex travel`, `delta`, `southwest air`, `united airlines`, `american airlines`, `airline`, `enterprise`, `hertz`, `avis`, `hotel`, `airbnb`, `expedia`, `marriott`, `hilton` -**Patterns**: -- `home depot` -- `lowes` -- `sawyer` -- `smart core` -- `garden` -- `hardware` -- `furniture` +### Groceries -**Examples**: -- "SAWYER + S* SMART CORE" -> Home & Garden +`h-e-b`, `heb`, `kroger`, `costco`, `wal-mart`, `walmart`, `wholefds`, `whole foods`, `sam's club`, `aldi`, `trader joe` ---- +### Dining Out -### Crypto Deposit -Cryptocurrency deposits and transfers +`benihana`, `golden corral`, `papa john`, `chuck e cheese`, `wingstop`, `cinemark`, `mcdonald`, `chick-fil-a`, `chipotle`, `starbucks`, `coffee`, `restaurant`, `grill`, `cafe`, `uber eats`, `doordash`, `whataburger`, `panda express`, `burger king`, `auntie anne`, `thai` -**Patterns**: -- `btc deposited` -- `bitcoin` -- `fidelity crypto` -- `eth deposited` -- `crypto` +### Credit Card Payment -**Examples**: -- "0.17713256 BTC deposited" -> Crypto Deposit -- "Fidelity Crypto® 8449251033" -> Crypto Deposit +`applecard`, `gsbapayment`, `chase payment`, `amex payment`, `discover payment`, `credit card payment`, `american express credit card`, `chase credit ca`, `wf credit card`, `apple credit card`, `amex epayment`, `chase credit card`, `thank you` ---- +### Giving -### Credit Card Payment -Credit card bill payments +`church`, `tithe`, `offering`, `ministry`, `missions` -**Patterns**: -- `applecard` -- `gsbapayment` -- `chase payment` -- `amex payment` -- `discover payment` +### Auto & Transport -**Examples**: -- "DIRECT DEBIT APPLECARD GSBAPAYMENT" -> Credit Card Payment +`tesla`, `supercha`, `vehicle registration`, `vehreg`, `dmv`, `parking`, `uber`, `lyft`, `shell`, `exxon`, `chevron`, `valero`, `gas station`, `toll`, `texaco`, `bp gas`, `car wash`, `safelite`, `auto glass` ---- +### Personal Care -### Exempt -Verification transactions, zero amounts +`salon`, `day spa`, `med spa`, `massage`, `barber`, `sephora`, `beauty supply`, `ulta`, `nail`, `hair`, `lash`, `wax` -**Patterns**: -- `ifacctverify` -- `verification` -- Amount = $0.00 or < $1.00 +### Health & Wellness -**Examples**: -- "WELLS FARGO IFACCTVERIFY" -> Exempt +`cvs`, `pharmacy`, `walgreens`, `doctor`, `medical`, `dental`, `clinic`, `hospital`, `urgent care` ---- +### Shopping -## Uncategorized +`marshalls`, `amazon`, `skims`, `tj maxx`, `ross`, `old navy`, `gap`, `nordstrom`, `macy`, `best buy`, `apple store`, `fashion nova`, `burlington`, `janie & jack`, `david yurman`, `dollar general` -Any transaction not matching the above patterns is marked as **"Uncategorized"** and flagged for manual review in the sync summary. +### Family Care -**Common uncategorized reasons**: -- New merchant not in patterns -- Unusual description format -- One-time or rare purchase +`daycare`, `childcare`, `kid`, `children`, `pediatric` -**To resolve**: Add the pattern to the appropriate category above. +### Bills & Utilities ---- +`autopay`, `acctverify`, `electric`, `water`, `internet`, `comcast`, `att`, `verizon`, `t-mobile`, `netflix`, `spotify`, `subscription`, `prime video` -## Pattern Matching Algorithm +### Cash Withdrawal -```python -def categorize_expense(description: str) -> str: - desc = description.lower() +`atm`, `cash withdrawal`, `cash advance` - # Check each category's patterns - for category, patterns in CATEGORY_PATTERNS.items(): - for pattern in patterns: - if pattern in desc: - return category +### Tuition - return "Uncategorized" +`university`, `college`, `tuition`, `coursera`, `udemy` -CATEGORY_PATTERNS = { - "Groceries": ["h-e-b", "heb", "kroger", "costco", "wal-mart", "walmart", ...], - "Dining Out": ["benihana", "golden corral", "papa john", ...], - "Auto & Transport": ["tesla", "supercha", "parking", "fastpark", ...], - # ... etc -} -``` +### Business Expense ---- - -## Extending Categories - -When the user wants to add new categories or patterns: - -1. **Add to existing category**: Update the patterns list above -2. **Create new category**: Add a new section with patterns and examples -3. **Expense Tracker sync**: Ensure the category name matches Budget Planner - -**Budget Planner categories** (must match exactly): -- Groceries -- Dining Out -- Auto & Transport -- Personal Care -- Health & Wellness -- Shopping -- Family Care -- Bills & Utilities -- Cash Withdrawal -- Tuition -- Business Expense -- Loan Payment -- Home & Garden -- Crypto Deposit -- Credit Card Payment -- Exempt -- Software & Tech -- Cell Phone -- Gas -- Water -- Light Bill -- Mortgage - ---- - -**Last Updated**: 2026-08-31 -**Maintainer**: Finance Guru TransactionSyncing skill - ---- - -## Ordering rules (2026-08-04) - -`CATEGORY_PATTERNS` is an ordered dict and **the first match wins**, so placement -is behavior, not style. Three orderings are load-bearing: - -1. **Travel before Credit Card Payment** so "American Express Travel" is a trip, - not a card payment. -2. **Credit Card Payment before Bills & Utilities.** A card autopay memo reads - "Chase Credit Cautopay", which contains the substring `autopay`. With Bills & - Utilities first, every card payment was mis-filed as a utility bill. -3. **`credit card payment` belongs to Credit Card Payment, not Loan Payment.** - It used to sit in Loan Payment, which is matched earlier, so card payments - never reached their own category. - -## Match against payee AND description - -The sync joins both fields before categorizing. `description` is the raw bank -memo (`DIRECT DEBIT AMEX EPAYMENT ACH PMT`); `payee` is SimpleFIN's normalized -merchant name (`American Express Credit Card`). Matching description alone left -67% of debit volume Uncategorized. **When adding a pattern, cover the -abbreviated memo form too**, for example `vehreg` alongside -`vehicle registration`, and `amex epayment` alongside `amex payment`. - -## Non-spend categories - -`NON_SPEND_CATEGORIES` in `categorize.py` holds `Transfer`, -`Credit Card Payment`, `Crypto Deposit`, `Exempt`, and `Retirement`. These move money rather -than consume it and **must be excluded from spend totals**. Including them -double-counts: once when a purchase hits the card, again when the card bill is -paid. On 2026-08-04 that inflated a 30-day review roughly 3x. - -## Categories added 2026-08-04 +`gumroad`, `ups`, `fedex`, `office depot`, `staples`, `postal`, `usps`, `linkedin`, `zoom`, `openai`, `chatgpt`, `anthropic`, `claude.ai`, `cursor`, `github`, `vercel`, `greptile`, `openrouter`, `slack`, `paddle`, `ui.com`, `ubiquiti`, `newegg`, `workspace` -### Giving -Church and ministry contributions. +### Loan Payment -**Patterns**: `anglicanchurch`, `church`, `tithe`, `offering`, `ministry`, `missions` +`loan payment`, `mortgage`, `mortg`, `mtgpmt`, `car payment`, `student loan` ### Fees & Interest -Card interest and bank charges, kept out of merchant spend. - -**Patterns**: `interest charge`, `finance charge`, `annual fee`, `late fee`, `overdraft`, `service charge`, `maintenance fee` - -### Additions to existing categories - -- **Transfer**: `transferred to`, `transfer to`, `transfer withdrawal`, `webxtransfer`, `wire transfer` -- **Credit Card Payment**: `credit card payment`, `american express credit card`, `chase credit ca`, `wf credit card`, `apple credit card`, `amex epayment`, `credit card` -- **Loan Payment**: `mortg`, `mtgpmt`, `aes stdnt` (bank memos abbreviate) -- **Auto & Transport**: `vehicle registration`, `vehreg`, `dmv` -- **Business Expense**: `anthropic`, `claude.ai`, `cursor`, `github`, `vercel` - -## Substring collisions fixed 2026-09-08 - -Patterns are plain substring matches, so a short pattern can fire inside a -longer, unrelated word. Three were silently misfiling real money: - -1. **`spa` matched `SPAXX`.** Fidelity's core position is the SPAXX money - market, so every core sweep, some of them five figures, was booked as - _Personal Care_. The bare pattern is gone; `day spa`, `med spa`, and - `massage` replace it, and a `spaxx` / `core account` guard returns `Exempt` - ahead of the table. -2. **`credit card` matched a store purchase.** SimpleFIN normalizes some - merchant payees to ` Credit Card`, so a Macy's shopping trip read - as a bill payment and was dropped from spend totals as non-spend. The bare - catch-all is gone. **Every pattern in Credit Card Payment must be - payment-specific.** -3. **`uber` claimed `Uber Eats`.** Food delivery was booking as a rideshare. - `uber eats` now sits in Dining Out, which is matched before Auto & Transport. - -**When adding a pattern shorter than about six characters, check it against a -real merchant list first.** Replaying the rules over the stored feed found -`spa` had also been capturing `GOOGLE WORKSPACE UNIF` (note the doubled space, -which is why the pattern is the bare word `workspace`) and `LIVING SPACES -MOBILE`. Removing a broad pattern is only safe once every merchant it had been -carrying has an explicit pattern of its own, so **always replay before and after -a rule change** rather than trusting the new patterns alone. -## The card-payment credit leg +`interest charge`, `finance charge`, `annual fee`, `annual membership fee`, `membership fee`, `late fee`, `overdraft`, `service charge`, `maintenance fee`, `maintenance charge`, `sie fee`, `adj redist`, `bounced check`, `returned check`, `nsf fee` -A bill payment posts twice: a debit on the funding account and a credit on the -card itself, worded `PAYMENT - THANK YOU` or `Payment Thank You`. Only the debit -was recognized, so the credit leg sat in Uncategorized and inflated any income -figure summed from credits. `thank you` now belongs to Credit Card Payment, -which is matched before Bills & Utilities and so also reclaims the -`AUTOPAY PAYMENT - THANK YOU` rows that `autopay` had taken. +### Entertainment -## Categories added 2026-09-08 +`playstation`, `bounce`, `gamestop`, `amc theat`, `ticketmaster` -### Entertainment -Recreation, kept separate from Dining Out so restaurants stay readable. - -**Patterns**: `playstation`, `andretti`, `bounce`, `gamestop`, `amc theat`, `ticketmaster` - -**Placed after Fees & Interest on purpose**, though position alone is not the -whole defense. An earlier category only wins if one of its patterns actually -matches, and `BOUNCED CHECK FEE` matched nothing in Fees & Interest while -containing `bounce`. Every returned-payment wording is therefore listed in Fees -& Interest explicitly. Caught by CodeRabbit on PR #176. - -### Business Income -Consulting revenue. **It has no pattern list.** The payer's name is private data -and stays out of this repository, so the account carries the signal: an -unmatched credit on an account matching `is_business_account()` is income. The -fallback runs _after_ the pattern table, so an explicit `Transfer` pattern still -claims an inter-account move first. Added to `BUSINESS_CATEGORIES`, so household -reviews exclude it and business P&L includes it. - -### Additions to existing categories - -- **Transfer**: `transfer deposit`, `transfer from`, `instant transfer`, `capital one bank` -- **Groceries**: `halal meat`, `southwest farmers` (before Dining Out, so the butcher does not match `halal guys`) -- **Dining Out**: `uber eats`, `doordash`, `whataburger`, `panda express`, `burger king`, `auntie anne`, `pick up stix`, `chicken salad chick`, `piada`, `nishiki`, `gringos`, `rouxpour`, `halal guys`, `broken egg`, `thai` -- **Auto & Transport**: `texaco`, `bp gas`, `car wash`, `parkify`, `safelite`, `auto glass` -- **Personal Care**: `clean skin`, `cloud 9 spa`, `lash`, `wax`, `face reality` -- **Health & Wellness**: `mychart`, `methodist` (Giving matches `church` first, so a congregation still lands in Giving) -- **Shopping**: `shopwss`, `fashion nova`, `burlington`, `uptown cheapskate`, `janie & jack`, `david yurman`, `dollar general` -- **Bills & Utilities**: `rhythm ops`, `total wireless`, `tidal`, `prime video` -- **Business Expense**: `greptile`, `openrouter`, `slack`, `paddle`, `ui.com`, `ubiquiti`, `newegg`, `bambula`, `connectech`, `pga frisco`, `workspace` -- **Fees & Interest**: `adj redist`, `bounced check`, `returned check`, `nsf fee` -- **Home & Garden**: `bermuda dude`, `living spaces`, `lawn`, `wayfair`, `flower shop` - -Homelab and fabrication hardware is business input, and the PGA Frisco charge is -a business trip. Both classified by the account owner 2026-09-08. +### Home & Garden -## Deliberately left Uncategorized +`home depot`, `lowes`, `garden`, `hardware`, `furniture`, `living spaces`, `lawn`, `wayfair` -`Paid Check` and `Target` are ambiguous by nature: a written check or a Target -run can be groceries, household, or shopping. Guessing is worse than a visible -gap. Review these by hand. +### Crypto Deposit -`School` is the same case, and so is any Apple Pay passthrough where the memo -carries only the wallet prefix and a merchant the owner has not identified. +`btc deposited`, `bitcoin`, `fidelity crypto`, `eth deposited`, `crypto` diff --git a/.claude/skills/TransactionSyncing/SKILL.md b/.claude/skills/TransactionSyncing/SKILL.md index 3401ef9f..3544c7ef 100644 --- a/.claude/skills/TransactionSyncing/SKILL.md +++ b/.claude/skills/TransactionSyncing/SKILL.md @@ -199,7 +199,10 @@ SnapTrade activities SimpleFIN dump (bun run src/dump.ts) ### Smart Categorization Categorization is executable and runs inside the expense adapter, so the -`category` column arrives pre-filled on every `bank_transactions` row. The rules +`category` column arrives pre-filled on every `bank_transactions` row. +Household merchants (local businesses, the daycare, the church) are private +and live in the instance as `merchant-rules.yaml`, merged into the public table +at sync time; the public table holds national brands only. The rules live in code at `src/integrations/simplefin/categorize.py` (`categorize_expense(text, amount)`), which is the source of truth mirroring the human-readable `CategoryRules.md`. Keep the two in sync when adding patterns. diff --git a/.gitignore b/.gitignore index 68362feb..a74c6827 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ notebooks/ # User's financial profile (any location in repo tree) fin-guru/data/user-profile.yaml **/user-profile.yaml +**/merchant-rules.yaml # Onboarding state and progress (may contain sensitive configuration) .onboarding-state.json # Setup progress tracking (user-specific) @@ -170,3 +171,7 @@ reports/ # Instance notes and meeting records (household data; live in the instance directory) /.memory/ /.dev/meeting-notes/ + +# graft's local graph cache: regenerable, and it mirrors pre-scrub source text. +/graft/ +.claude/skills/graft/ diff --git a/PRIVACY.md b/PRIVACY.md index 5781b3a7..5d5dd93b 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -36,6 +36,7 @@ Finance Guru is a **private, single-user family-office system**. This document d - Raw Fidelity CSVs (gitignored) - SSNs, account numbers, and API keys (scrubbed by `src/utils/log.py:ScrubPIIProcessor` before they reach any log) - `.env` contents +- `merchant-rules.yaml`, your household merchant names (the engine ships only national brands) - `finance-guru-desktop/` runtime state (also gitignored) ## Security checklist (before publishing the fork) diff --git a/config/fidelity-mapping.json b/config/fidelity-mapping.json index e769c15a..7c4ce97f 100644 --- a/config/fidelity-mapping.json +++ b/config/fidelity-mapping.json @@ -271,21 +271,14 @@ "notes": "Credit-card payoff. The spending was already captured at the merchant via the CC statement. Counting these as expenses double-counts spending. Only treat as 'spending' if you are NOT also importing CC statements." }, "AUTO_LOAN_PAYMENT": { - "patterns": ["DIRECT DEBIT WELLS FARGO AUDRAFT"], + "patterns": ["AUDRAFT"], "sign": "negative", "true_spending": true, "expense_category": "Loan Payment", - "notes": "Wells Fargo 'AUDRAFT' = auto loan payment (not overdraft). Real recurring expense — keep separate from CC pass-throughs." + "notes": "'AUDRAFT' = auto loan payment (not overdraft). Real recurring expense — keep separate from CC pass-throughs." }, "DIRECT_DEBIT_BILL": { "patterns": [ - "DIRECT DEBIT AES STDNT LOAN", - "DIRECT DEBIT REGENT UNIVERS", - "DIRECT DEBIT BMOBNK", - "DIRECT DEBIT CPENERGY", - "DIRECT DEBIT BRAZORIA", - "DIRECT DEBIT SAFEPOINT", - "DIRECT DEBIT CLICKPAY", "DIRECT DEBIT" ], "sign": "negative", diff --git a/docs-site/src/content/docs/reference/instance-layout.md b/docs-site/src/content/docs/reference/instance-layout.md index aca55573..b0307568 100644 --- a/docs-site/src/content/docs/reference/instance-layout.md +++ b/docs-site/src/content/docs/reference/instance-layout.md @@ -19,6 +19,7 @@ The root is the value of `FIN_GURU_DATA_ROOT` when that variable is set and non- | `user-profile.yaml` | User profile. | | `config.yaml` | Instance configuration. | | `snaptrade-accounts.yaml` | [SnapTrade](https://snaptrade.com/) account-routing configuration. | +| `merchant-rules.yaml` | Household merchant patterns for expense categorization, merged into the engine's public brand table at sync time. | | `system-context.md` | Generated system context. | | `family_office.db` | Default family-office [SQLite](https://www.sqlite.org/) database. | | `dividend-schedules.yaml` | Dividend schedule file. | diff --git a/src/cli/instance_init.py b/src/cli/instance_init.py index 03c3cf82..ff4e80d9 100644 --- a/src/cli/instance_init.py +++ b/src/cli/instance_init.py @@ -31,6 +31,21 @@ __pycache__/ """ +MERCHANT_RULES = """# Household merchant patterns for expense categorization. +# +# The engine's public table only knows national brands. Your own merchants (the +# local dry cleaner, the daycare, the church) go here so they never leave this +# directory. Each key is a category from src/integrations/simplefin/categorize.py +# and each value is a list of lowercase substrings matched against the joined +# payee and bank memo. Within a category, order does not matter; across +# categories, the engine's table order decides ties. +# +# Groceries: +# - corner market +# Family Care: +# - little sprouts daycare +""" + USER_PROFILE = """# Add instance-specific values after creating the instance. system_ownership: {} orientation_status: {} @@ -268,6 +283,7 @@ def _build_plan( ), PlanStep(paths.env_file, _write_text(env_content)), PlanStep(paths.profile, _write_text(USER_PROFILE)), + PlanStep(paths.merchant_rules, _write_text(MERCHANT_RULES)), ) ) if not plugin_mode: diff --git a/src/config/instance_paths.py b/src/config/instance_paths.py index 03c49112..a20054f0 100644 --- a/src/config/instance_paths.py +++ b/src/config/instance_paths.py @@ -54,6 +54,11 @@ def snaptrade_accounts(self) -> Path: """Return the SnapTrade account-routing configuration path.""" return self.root / "snaptrade-accounts.yaml" + @property + def merchant_rules(self) -> Path: + """Return the household merchant-rules file for expense categorization.""" + return self.root / "merchant-rules.yaml" + @property def system_context(self) -> Path: """Return the generated system context path.""" diff --git a/src/integrations/refresh_all.py b/src/integrations/refresh_all.py index aa6fedad..bb061284 100644 --- a/src/integrations/refresh_all.py +++ b/src/integrations/refresh_all.py @@ -23,11 +23,17 @@ def refresh(database_url: str | None, *, months: int = 12) -> dict[str, Any]: Returns: Per-source statuses and the refresh timestamp. """ - account_config = InstancePaths.resolve().snaptrade_accounts + paths = InstancePaths.resolve() + account_config = paths.snaptrade_accounts operations: tuple[tuple[str, Callable[[], dict[str, Any]]], ...] = ( ("positions", lambda: sync_positions(account_config, database_url)), ("transactions", lambda: sync_transactions(account_config, database_url)), - ("expenses", lambda: sync_expenses(database_url, months=months)), + ( + "expenses", + lambda: sync_expenses( + database_url, months=months, merchant_rules=paths.merchant_rules + ), + ), ) sources: list[dict[str, Any]] = [] for name, operation in operations: diff --git a/src/integrations/simplefin/categorize.py b/src/integrations/simplefin/categorize.py index 7913ffb0..c1c54c56 100644 --- a/src/integrations/simplefin/categorize.py +++ b/src/integrations/simplefin/categorize.py @@ -1,14 +1,25 @@ -"""Categorize SimpleFIN transactions using ordered merchant patterns.""" +"""Categorize SimpleFIN transactions using ordered merchant patterns. + +The table in this module holds only national brands and generic wording. A +household's own merchants (the local dry cleaner, the daycare, the church) are +private data and live in the instance directory as ``merchant-rules.yaml``, +loaded at sync time by :func:`load_merchant_rules` and appended to the public +table by :func:`merge_patterns`. +""" import os +from collections.abc import Hashable, Mapping +from pathlib import Path + +import yaml # Order matters: the first matching category wins. Transfer and Travel come # first so an explicit remittance is not read as a merchant, and so # "American Express Travel" lands in Travel rather than Credit Card Payment. CATEGORY_PATTERNS: dict[str, tuple[str, ...]] = { "Transfer": ( - "taptap send", "amex send", + "taptap send", "cashed check", "zelle", "venmo", @@ -24,8 +35,6 @@ "instant transfer", "webxtransfer", "wire transfer", - # Inbound movement between the owner's own Capital One accounts. - "capital one bank", ), "Travel": ( "american express travel", @@ -53,13 +62,9 @@ "walmart", "wholefds", "whole foods", - "makola", "sam's club", "aldi", "trader joe", - # Precedes Dining Out so the butcher does not match "halal guys". - "halal meat", - "southwest farmers", ), "Dining Out": ( "benihana", @@ -76,8 +81,6 @@ "restaurant", "grill", "cafe", - "makiin", - "sparkly photo", # Must precede Auto & Transport, whose "uber" pattern would otherwise # book a food delivery as a rideshare. "uber eats", @@ -86,14 +89,6 @@ "panda express", "burger king", "auntie anne", - "pick up stix", - "chicken salad chick", - "piada", - "nishiki", - "gringos", - "rouxpour", - "halal guys", - "broken egg", "thai", ), # Must precede Bills & Utilities: a card autopay string such as @@ -121,7 +116,6 @@ # totals as non-spend. Keep every pattern here payment-specific. ), "Giving": ( - "anglicanchurch", "church", "tithe", "offering", @@ -135,20 +129,17 @@ "vehreg", "dmv", "parking", - "fastpark", "uber", "lyft", "shell", "exxon", "chevron", "valero", - "buc-ee", "gas station", "toll", "texaco", "bp gas", "car wash", - "parkify", "safelite", "auto glass", ), @@ -162,33 +153,22 @@ "barber", "sephora", "beauty supply", - "supreme beauty", "ulta", "nail", "hair", - "shaving grace", - "cash app", - "clean skin", - "cloud 9 spa", "lash", "wax", - "face reality", ), "Health & Wellness": ( "cvs", "pharmacy", "walgreens", - "life time", "doctor", "medical", "dental", "clinic", "hospital", "urgent care", - "mychart", - # Giving matches "church" first, so a Methodist congregation still - # lands in Giving and only the outpatient centers arrive here. - "methodist", ), "Shopping": ( "marshalls", @@ -202,18 +182,13 @@ "macy", "best buy", "apple store", - "shopwss", "fashion nova", "burlington", - "uptown cheapskate", "janie & jack", "david yurman", "dollar general", ), "Family Care": ( - "aqua tots", - "brightwheel", - "brghtwhl", "daycare", "childcare", "kid", @@ -233,14 +208,10 @@ "netflix", "spotify", "subscription", - "rhythm ops", - "total wireless", - "tidal", "prime video", ), "Cash Withdrawal": ("atm", "cash withdrawal", "cash advance"), "Tuition": ( - "regent univer", "university", "college", "tuition", @@ -268,14 +239,9 @@ "openrouter", "slack", "paddle", - # Homelab and fabrication hardware. Classified as business input by the - # account owner 2026-09-08, alongside the existing developer tooling. "ui.com", "ubiquiti", "newegg", - "bambula", - "connectech", - "pga frisco", # "GOOGLE WORKSPACE" contains "spa" and was landing in Personal Care. # Matched on the bare word: the bank memo doubles the space after GOOGLE. "workspace", @@ -286,11 +252,10 @@ "Loan Payment": ( "loan payment", "mortgage", - "mortg", # Fidelity/Truist abbreviate: "TRUIST MORTG TEL MTGPMT" + "mortg", # bank memos abbreviate: "ANYBANK MORTG TEL MTGPMT" "mtgpmt", "car payment", "student loan", - "aes stdnt", ), "Fees & Interest": ( "interest charge", @@ -317,7 +282,6 @@ # returned-payment wording is listed there explicitly. "Entertainment": ( "playstation", - "andretti", "bounce", "gamestop", "amc theat", @@ -326,17 +290,13 @@ "Home & Garden": ( "home depot", "lowes", - "sawyer", - "smart core", "garden", "hardware", "furniture", - "bermuda dude", # "LIVING SPACES" contains "spa" and was landing in Personal Care. "living spaces", "lawn", "wayfair", - "flower shop", ), "Crypto Deposit": ( "btc deposited", @@ -347,6 +307,94 @@ ), } +PatternTable = dict[str, tuple[str, ...]] + + +class MerchantRulesError(ValueError): + """The instance merchant-rules file is malformed.""" + + +class _UniqueKeyLoader(yaml.SafeLoader): + """SafeLoader that rejects a repeated mapping key. + + Plain ``safe_load`` keeps only the last value for a repeated key, so a + second ``Groceries:`` block would silently drop the first one's patterns. + """ + + def construct_mapping( + self, node: yaml.MappingNode, deep: bool = False + ) -> dict[object, object]: + seen: set[object] = set() + for key_node, _ in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, Hashable): + raise yaml.constructor.ConstructorError( + None, None, "found unhashable key", key_node.start_mark + ) + if key in seen: + raise yaml.constructor.ConstructorError( + None, None, f"duplicate key {key!r}", key_node.start_mark + ) + seen.add(key) + return super().construct_mapping(node, deep=deep) + + +def load_merchant_rules(path: Path) -> PatternTable: + """Read household merchant patterns from the instance. + + The file maps a category name to a list of lowercase substrings, for + example ``Family Care: [little sprouts daycare]``. A missing file means + the household has no private rules yet and is not an error. + + Raises: + MerchantRulesError: On a category the public table does not define, a + category listed twice, or a value that is not a list of strings, + so a typo blocks the sync instead of silently dropping rules. + """ + if not path.is_file(): + return {} + try: + text = path.read_text(encoding="utf-8") + loaded = yaml.load(text, Loader=_UniqueKeyLoader) + except yaml.YAMLError as exc: + raise MerchantRulesError(f"{path} is not valid YAML: {exc}") from exc + if loaded is None: + return {} + if not isinstance(loaded, dict): + raise MerchantRulesError(f"{path} must be a mapping of category to patterns") + rules: PatternTable = {} + for category, patterns in loaded.items(): + if category not in CATEGORY_PATTERNS: + raise MerchantRulesError( + f"{path}: unknown category {category!r}; " + f"choose one of {', '.join(CATEGORY_PATTERNS)}" + ) + if patterns is None: + continue + if not isinstance(patterns, list) or not all( + isinstance(item, str) and item.strip() for item in patterns + ): + raise MerchantRulesError( + f"{path}: {category!r} must be a list of non-empty strings" + ) + rules[category] = tuple(item.strip().lower() for item in patterns) + return rules + + +def merge_patterns( + base: Mapping[str, tuple[str, ...]], extra: Mapping[str, tuple[str, ...]] +) -> PatternTable: + """Append household patterns to each category of the public table. + + Category order, and so match precedence, comes from ``base``. Household + patterns only extend a category; they never create or reorder one. + """ + return { + category: patterns + tuple(extra.get(category, ())) + for category, patterns in base.items() + } + + # Categories that move money rather than consume it. Expense reviews should # exclude these from spend totals, otherwise a card purchase is counted twice: # once on the card account and again when the card bill is paid. @@ -426,6 +474,7 @@ def categorize_expense( text: str | None, amount: float | None = None, account_name: str | None = None, + patterns: Mapping[str, tuple[str, ...]] | None = None, ) -> str: """Return the first matching expense category. @@ -435,6 +484,9 @@ def categorize_expense( account_name: Owning account name. Required to distinguish business payroll from personal spending, because identical memo text ("Paid Check") means different things by account. + patterns: Pattern table to match against. Defaults to the public + table; the sync passes the public table merged with the + household's ``merchant-rules.yaml``. Returns: The matching category, ``Exempt``, or ``Uncategorized``. @@ -467,8 +519,9 @@ def categorize_expense( ): return "Payroll" - for category, patterns in CATEGORY_PATTERNS.items(): - if any(pattern in normalized for pattern in patterns): + table = CATEGORY_PATTERNS if patterns is None else patterns + for category, category_patterns in table.items(): + if any(pattern in normalized for pattern in category_patterns): return category # Unmatched money arriving in a business account is revenue. Running this diff --git a/src/integrations/simplefin/sync_expenses_db.py b/src/integrations/simplefin/sync_expenses_db.py index a0115446..c4210e0e 100644 --- a/src/integrations/simplefin/sync_expenses_db.py +++ b/src/integrations/simplefin/sync_expenses_db.py @@ -13,7 +13,13 @@ from typing import Any from src.config.instance_paths import InstancePaths, _db_path, load_instance_env -from src.integrations.simplefin.categorize import categorize_expense +from src.integrations.simplefin.categorize import ( + CATEGORY_PATTERNS, + PatternTable, + categorize_expense, + load_merchant_rules, + merge_patterns, +) class SimpleFinSyncError(RuntimeError): @@ -160,7 +166,10 @@ def resolve_direction(text: str | None, amount: float | None) -> str: def normalize_transaction( - account: dict[str, Any], txn: dict[str, Any], now: str + account: dict[str, Any], + txn: dict[str, Any], + now: str, + patterns: PatternTable | None = None, ) -> tuple[Any, ...]: """Normalize a SimpleFIN transaction into database column order. @@ -168,6 +177,8 @@ def normalize_transaction( account: SimpleFIN account containing the transaction. txn: SimpleFIN transaction payload. now: ISO timestamp for the current sync. + patterns: Category table, the public table merged with the household + rules. Defaults to the public table alone. Returns: A tuple matching the ``bank_transactions`` column order. @@ -202,7 +213,7 @@ def normalize_transaction( txn.get("description"), amount, direction, - categorize_expense(category_text, amount, account.get("name")), + categorize_expense(category_text, amount, account.get("name"), patterns), now, ) @@ -213,6 +224,7 @@ def sync( months: int = 12, app_dir: Path = DEFAULT_APP_DIR, dump_provider: Callable[[int, Path], dict[str, Any]] = run_dump, + merchant_rules: Path | None = None, ) -> dict[str, Any]: """Sync SimpleFIN transactions into the shared SQLite database. @@ -221,14 +233,21 @@ def sync( months: Number of months of history to request. app_dir: Directory containing the SimpleFIN Bun application. dump_provider: Callable that returns a SimpleFIN account-set payload. + merchant_rules: Household ``merchant-rules.yaml``; ``None`` categorizes + with the public table alone. Returns: Counts and metadata describing the completed sync. Raises: SimpleFinSyncError: If the provider returns an invalid payload. + MerchantRulesError: If ``merchant_rules`` is malformed. """ db = _db_path(database_url) + patterns = merge_patterns( + CATEGORY_PATTERNS, + load_merchant_rules(merchant_rules) if merchant_rules else {}, + ) now = datetime.now(UTC).replace(microsecond=0).isoformat() payload = dump_provider(months, app_dir) if not isinstance(payload, dict) or "accounts" not in payload: @@ -257,7 +276,7 @@ def sync( if not txn.get("id"): skipped_missing_id += 1 continue - row = normalize_transaction(account, txn, now) + row = normalize_transaction(account, txn, now, patterns) rows.append(row) category = str(row[10]) by_category[category] = by_category.get(category, 0) + 1 @@ -345,7 +364,9 @@ def main(argv: list[str] | None = None) -> int: return 0 try: - summary = sync(database_url, months=args.months) + summary = sync( + database_url, months=args.months, merchant_rules=paths.merchant_rules + ) except (SimpleFinSyncError, ValueError, RuntimeError, OSError) as exc: print(f"Sync error: {exc}", file=sys.stderr) return 1 diff --git a/tests/python/fixtures/fidelity_history_golden.csv b/tests/python/fixtures/fidelity_history_golden.csv index e8efffb7..989317e6 100644 --- a/tests/python/fixtures/fidelity_history_golden.csv +++ b/tests/python/fixtures/fidelity_history_golden.csv @@ -7,4 +7,4 @@ Run Date,Action,Symbol,Description,Type,Price ($),Quantity,Commission ($),Fees ( 05/02/2026,"REINVESTMENT as of Apr-30-2026 J P MORGAN EXCHANGE TRADED FD EQUITY... (JEPI) (Cash)",JEPI,"J P MORGAN EXCHANGE TRADED FD EQUITY",Cash,55.00,1.818,,,,-100.00,-4150.00, 05/01/2026,"JOURNALED JNL VS A/C TYPES as of May-01-2026 (Cash)",,"No Description",Cash,,0.000,,,,200.00,-4050.00, 04/30/2026,"DIRECT DEBIT APPLECARD GSBAPAYMENT (Cash)", ,"No Description",Cash,,0.000,,,,-1500.00,-4250.00, -04/29/2026,"DEBIT CARD PURCHASE H-E-B #458 PEARLAND TX 042926 AUTHID:123456 (Cash)",,"No Description",Cash,,0.000,,,,-75.00,-2750.00, +04/29/2026,"DEBIT CARD PURCHASE H-E-B #458 ANYTOWN TX 042926 AUTHID:123456 (Cash)",,"No Description",Cash,,0.000,,,,-75.00,-2750.00, diff --git a/tests/python/test_categorize.py b/tests/python/test_categorize.py index dd8ad4c7..32388fb9 100644 --- a/tests/python/test_categorize.py +++ b/tests/python/test_categorize.py @@ -5,13 +5,20 @@ credit card payments to Loan Payment. """ +import re +from pathlib import Path + import pytest from src.integrations.simplefin.categorize import ( BUSINESS_CATEGORIES, + CATEGORY_PATTERNS, NON_SPEND_CATEGORIES, + MerchantRulesError, categorize_expense, is_business_account, + load_merchant_rules, + merge_patterns, ) @@ -128,7 +135,7 @@ class TestTransfers: [ "Transferred to Z Cash", "Transfer to brokerage", - "DIRECT DEBIT BMOBNK CK WEBXTRANSFER", + "DIRECT DEBIT ANYBANK CK WEBXTRANSFER", "Outgoing wire transfer", ], ) @@ -154,27 +161,27 @@ class TestCreditCardPayments: ], ) def test_card_payments_are_credit_card_payment(self, text: str) -> None: - assert categorize_expense(text, -9402.47) == "Credit Card Payment" + assert categorize_expense(text, -9400.00) == "Credit Card Payment" def test_real_loans_still_route_to_loan_payment(self) -> None: - assert categorize_expense("DIRECT DEBIT AES STDNT LOAN", -4169.25) == ( + assert categorize_expense("DIRECT DEBIT STUDENT LOAN SVC", -4200.00) == ( "Loan Payment" ) - assert categorize_expense("TRUIST MORTG TEL MTGPMT", -1500.00) == ( + assert categorize_expense("ANYBANK MORTG TEL MTGPMT", -1500.00) == ( "Loan Payment" ) def test_amex_travel_still_beats_card_payment(self) -> None: """Travel is matched before Credit Card Payment on purpose.""" - assert categorize_expense("American Express Travel", -813.44) == "Travel" + assert categorize_expense("American Express Travel", -800.00) == "Travel" class TestNewCategories: def test_church_giving(self) -> None: - assert categorize_expense("Anglicanchurchofpentx", -100.00) == "Giving" + assert categorize_expense("First Church of Anytown", -100.00) == "Giving" def test_interest_charge_is_a_fee(self) -> None: - assert categorize_expense("Interest Charge", -267.86) == "Fees & Interest" + assert categorize_expense("Interest Charge", -270.00) == "Fees & Interest" def test_vehicle_registration_is_transport(self) -> None: assert ( @@ -183,7 +190,7 @@ def test_vehicle_registration_is_transport(self) -> None: ) def test_ai_tooling_is_a_business_expense(self) -> None: - assert categorize_expense("Anthropic", -210.80) == "Business Expense" + assert categorize_expense("Anthropic", -210.00) == "Business Expense" class TestRawBankMemos: @@ -203,7 +210,7 @@ class TestRawBankMemos: ("Transfer", "TRANSFER WITHDRAWAL To ....2222", "Transfer"), ( "State of Texas Vehicle Registration", - "BRAZORIA VEHREG 1302ANGLETON TX", + "ANYCOUNTY VEHREG 1302ANYTOWN TX", "Auto & Transport", ), ], @@ -215,10 +222,10 @@ def test_joined_payee_and_description( def test_raw_memo_alone_still_matches(self) -> None: """Even without the payee, the abbreviated memo must categorize.""" - assert categorize_expense("DIRECT DEBIT AMEX EPAYMENT ACH PMT", -9402.47) == ( + assert categorize_expense("DIRECT DEBIT AMEX EPAYMENT ACH PMT", -9400.00) == ( "Credit Card Payment" ) - assert categorize_expense("BRAZORIA VEHREG 1302ANGLETON TX", -272.00) == ( + assert categorize_expense("ANYCOUNTY VEHREG 1302ANYTOWN TX", -272.00) == ( "Auto & Transport" ) @@ -233,8 +240,8 @@ def test_membership(self) -> None: def test_double_count_case_is_excluded(self) -> None: """An Amex purchase and the Amex bill payment must not both count.""" - purchase = categorize_expense("Amazon", -329.83) - bill = categorize_expense("American Express Credit Card", -9402.47) + purchase = categorize_expense("Amazon", -330.00) + bill = categorize_expense("American Express Credit Card", -9400.00) assert purchase not in NON_SPEND_CATEGORIES assert bill in NON_SPEND_CATEGORIES @@ -250,7 +257,7 @@ class TestUnchangedBehaviour: ("Tesla Supercharger", "Auto & Transport"), ("CVS Pharmacy", "Health & Wellness"), ("Amazon", "Shopping"), - ("Brightwheel", "Family Care"), + ("Sunny Days Daycare", "Family Care"), ("Netflix subscription", "Bills & Utilities"), ("ATM cash withdrawal", "Cash Withdrawal"), ], @@ -276,7 +283,7 @@ class TestSubstringCollisions: ) def test_spaxx_sweep_is_exempt_not_personal_care(self, memo: str) -> None: """ "SPAXX" contains "spa", so every core sweep was booked as a spa visit.""" - assert categorize_expense(memo, 7716.47, "Cash Management (4444)") == "Exempt" + assert categorize_expense(memo, 7700.00, "Cash Management (4444)") == "Exempt" def test_real_spa_still_reaches_personal_care(self) -> None: assert categorize_expense("SERENITY DAY SPA", -120.00, None) == "Personal Care" @@ -287,8 +294,8 @@ def test_store_purchase_is_not_a_card_payment(self) -> None: bill payment, which excluded a real purchase from spend totals.""" assert ( categorize_expense( - "Macy's Credit Card MACYS PEARLAND TWN CTR", - -255.42, + "Macy's Credit Card MACYS ANYTOWN TWN CTR", + -255.00, "Sapphire Preferred (3333)", ) == "Shopping" @@ -296,14 +303,14 @@ def test_store_purchase_is_not_a_card_payment(self) -> None: def test_actual_card_payment_still_classified(self) -> None: assert ( - categorize_expense("Chase Credit Card CHASE CREDIT CRD", -38.19, None) + categorize_expense("Chase Credit Card CHASE CREDIT CRD", -38.00, None) == "Credit Card Payment" ) def test_food_delivery_is_dining_not_rideshare(self) -> None: """ "Uber Eats" contains "uber"; Auto & Transport was claiming it.""" assert categorize_expense("Uber Eats", -1.84, None) == "Dining Out" - assert categorize_expense("Uber", -36.94, None) == "Auto & Transport" + assert categorize_expense("Uber", -37.00, None) == "Auto & Transport" class TestCardPaymentCreditLeg: @@ -321,7 +328,7 @@ class TestCardPaymentCreditLeg: ], ) def test_thank_you_credit_is_a_card_payment(self, memo: str) -> None: - assert categorize_expense(memo, 20009.65, "Platinum Card® (3333)") == ( + assert categorize_expense(memo, 20000.00, "Platinum Card® (3333)") == ( "Credit Card Payment" ) @@ -337,12 +344,12 @@ class TestBusinessIncome: PERSONAL = "360 Checking (2222)" def test_unmatched_credit_on_business_account_is_income(self) -> None: - assert categorize_expense("CGSOPERATING", 12320.00, self.BIZ) == ( + assert categorize_expense("ACME OPERATING", 12000.00, self.BIZ) == ( "Business Income" ) def test_unmatched_credit_on_personal_account_is_not_income(self) -> None: - assert categorize_expense("SOME MERCHANT", 12320.00, self.PERSONAL) == ( + assert categorize_expense("SOME MERCHANT", 12000.00, self.PERSONAL) == ( "Uncategorized" ) @@ -369,15 +376,11 @@ class TestEntertainment: ("memo", "expected"), [ ("PlayStation Network", "Entertainment"), - ("CE ANDRETTIS", "Entertainment"), ("Bounce", "Entertainment"), - # The account owner classified the PGA Frisco charge as a business - # trip on 2026-09-08, so it belongs to the business bucket. - ("PGA FRISCO FRONT", "Business Expense"), ], ) def test_recreation_patterns(self, memo: str, expected: str) -> None: - assert categorize_expense(memo, -292.24, None) == expected + assert categorize_expense(memo, -290.00, None) == expected @pytest.mark.parametrize( "memo", @@ -412,7 +415,6 @@ class TestSpaPatternFallout: "Sapphire Preferred (3333)", "Home & Garden", ), - ("CLOUD 9 SPA PEARLAND", "Rewards Card (5555)", "Personal Care"), ], ) def test_merchants_the_spa_pattern_had_captured( @@ -420,7 +422,7 @@ def test_merchants_the_spa_pattern_had_captured( ) -> None: """Removing the bare "spa" pattern must not drop these into Uncategorized: each one needs its own explicit pattern.""" - assert categorize_expense(memo, -202.75, account) == expected + assert categorize_expense(memo, -200.00, account) == expected def test_inbound_transfer_is_not_business_income(self) -> None: """ "Instant Transfer Received From ...." does not contain "transfer @@ -433,3 +435,73 @@ def test_inbound_transfer_is_not_business_income(self) -> None: ) == "Transfer" ) + + +class TestMerchantRules: + """Household merchants live in the instance, never in the public table.""" + + def test_missing_file_means_no_household_rules(self, tmp_path) -> None: + assert load_merchant_rules(tmp_path / "merchant-rules.yaml") == {} + + def test_rules_extend_a_category_without_reordering(self, tmp_path) -> None: + rules = tmp_path / "merchant-rules.yaml" + rules.write_text( + "Groceries:\n - Corner Market\nFamily Care:\n - little sprouts\n" + ) + merged = merge_patterns(CATEGORY_PATTERNS, load_merchant_rules(rules)) + assert list(merged) == list(CATEGORY_PATTERNS) + assert merged["Groceries"][-1] == "corner market" + assert ( + categorize_expense("CORNER MARKET #12", -40.00, None, merged) == "Groceries" + ) + assert ( + categorize_expense("LITTLE SPROUTS", -40.00, None, merged) == "Family Care" + ) + # The public table alone still knows nothing about the household. + assert categorize_expense("LITTLE SPROUTS", -40.00, None) == "Uncategorized" + + def test_public_table_order_still_decides_ties(self, tmp_path) -> None: + """A household pattern in a later category cannot outrank an earlier + category's match; precedence stays with the public table.""" + rules = tmp_path / "merchant-rules.yaml" + rules.write_text("Shopping:\n - church store\n") + merged = merge_patterns(CATEGORY_PATTERNS, load_merchant_rules(rules)) + assert categorize_expense("CHURCH STORE", -20.00, None, merged) == "Giving" + + @pytest.mark.parametrize( + "body", + [ + "Snacks:\n - corner market\n", + "Groceries: corner market\n", + "Groceries:\n - ''\n", + "- Groceries\n", + "Groceries:\n - corner market\nGroceries:\n - farm stand\n", + "[]\n", + "false\n", + "? [Groceries, Shopping]\n: [corner market]\n", + ], + ) + def test_malformed_rules_block_instead_of_dropping( + self, tmp_path, body: str + ) -> None: + rules = tmp_path / "merchant-rules.yaml" + rules.write_text(body) + with pytest.raises(MerchantRulesError): + load_merchant_rules(rules) + + +def test_category_rules_doc_mirrors_the_public_table() -> None: + """CategoryRules.md lists every public pattern per category, in table order.""" + doc = ( + Path(__file__).resolve().parents[2] + / ".claude" + / "skills" + / "TransactionSyncing" + / "CategoryRules.md" + ).read_text(encoding="utf-8") + listed = doc.split("## Public table", 1)[1] + documented: dict[str, tuple[str, ...]] = {} + for block in listed.split("### ")[1:]: + heading, _, body = block.partition("\n") + documented[heading.strip()] = tuple(re.findall(r"`([^`]+)`", body)) + assert documented == CATEGORY_PATTERNS diff --git a/tests/python/test_instance_init.py b/tests/python/test_instance_init.py index 06a46d4b..1c0572a4 100644 --- a/tests/python/test_instance_init.py +++ b/tests/python/test_instance_init.py @@ -170,6 +170,7 @@ def test_fresh_root_creates_complete_instance(tmp_path: Path) -> None: root / ".gitignore", paths.env_file, paths.profile, + paths.merchant_rules, root / "AGENTS.md", root / "CLAUDE.md", ) diff --git a/tests/python/test_simplefin_expenses.py b/tests/python/test_simplefin_expenses.py index b0cf78ac..af187df0 100644 --- a/tests/python/test_simplefin_expenses.py +++ b/tests/python/test_simplefin_expenses.py @@ -44,7 +44,7 @@ def test_default_app_dir_is_checkout_owned(tmp_path, monkeypatch) -> None: "id": "TXN-pending", "posted": 0, "payee": "H-E-B", - "description": "H-E-B #063 PEARLAND TX", + "description": "H-E-B #063 ANYTOWN TX", "amount": "12.34", }, ], @@ -54,7 +54,7 @@ def test_default_app_dir_is_checkout_owned(tmp_path, monkeypatch) -> None: def test_categorize_expense_patterns_and_exemptions() -> None: - assert categorize_expense("H-E-B #063 PEARLAND TX") == "Groceries" + assert categorize_expense("H-E-B #063 ANYTOWN TX") == "Groceries" assert categorize_expense("CHUCK E CHEESE 578") == "Dining Out" assert categorize_expense("Tesla SUPERCHA 123") == "Auto & Transport" assert categorize_expense("OPENAI *CHATGPT SUBSCR") == "Business Expense" @@ -73,7 +73,7 @@ def test_resolve_direction_prefers_feed_wording_over_sign() -> None: == "credit" ) assert ( - resolve_direction("DIRECT DEPOSIT Ridgeline SerPAYROLL (Cash)", -2273.03) + resolve_direction("DIRECT DEPOSIT Ridgeline SerPAYROLL (Cash)", -2300.00) == "credit" ) assert ( @@ -83,7 +83,7 @@ def test_resolve_direction_prefers_feed_wording_over_sign() -> None: # A debit marker with an unexpectedly positive amount still reads as outflow. assert resolve_direction("DIRECT DEBIT CHASE CREDIT CAUTOPAY", 600.00) == "debit" # No marker falls back to sign. - assert resolve_direction("H-E-B #063 PEARLAND TX", -54.13) == "debit" + assert resolve_direction("H-E-B #063 ANYTOWN TX", -54.13) == "debit" assert resolve_direction("Capital One", 2000.00) == "credit" assert resolve_direction(None, None) == "credit" @@ -181,7 +181,7 @@ def test_resolve_direction_handles_fidelity_card_and_transfer_wording() -> None: # Outflows the feed reported as positive. assert ( resolve_direction( - "CASH ADVANCE *SEDONA LAKES MANVEL TX 081826 AUTHID:624330 (Cash)", 654.00 + "CASH ADVANCE *LAKESIDE ANYTOWN TX 081826 AUTHID:624330 (Cash)", 654.00 ) == "debit" ) @@ -193,7 +193,7 @@ def test_resolve_direction_handles_fidelity_card_and_transfer_wording() -> None: ) assert ( resolve_direction( - "DEBIT CARD PURCHASE CASH APP*ANGLICAN CHUR cash.app TX 082726", 20.00 + "DEBIT CARD PURCHASE CASH APP*FIRST CHURCH cash.app TX 082726", 20.00 ) == "debit" ) @@ -227,9 +227,9 @@ def test_retirement_account_activity_is_not_household_spending() -> None: too generic to match on (a charitable contribution is real spending). """ retirement = "EXAMPLE EMPLOYER 401(K) RETIREMENT PLAN" - assert categorize_expense("contribution", -123.08, retirement) == "Retirement" - assert categorize_expense("contribution", -30.77, retirement) == "Retirement" - assert categorize_expense("dividend", 7.84, retirement) == "Retirement" + assert categorize_expense("contribution", -120.00, retirement) == "Retirement" + assert categorize_expense("contribution", -30.00, retirement) == "Retirement" + assert categorize_expense("dividend", 8.00, retirement) == "Retirement" # Retirement must be excluded from household spending totals. assert "Retirement" in NON_SPEND_CATEGORIES # A contribution on a normal account is still ordinary spending, not Retirement. @@ -252,20 +252,58 @@ def test_inbound_direct_deposits_are_payroll_regardless_of_employer_memo() -> No """ cma = "Cash Management (Joint WROS) (0001)" assert ( - categorize_expense("DIRECT DEPOSIT ACME STAFFINGDIR DEP (Cash)", 2207.38, cma) + categorize_expense("DIRECT DEPOSIT ACME STAFFINGDIR DEP (Cash)", 2200.00, cma) == "Payroll" ) assert ( - categorize_expense("DIRECT DEPOSIT Ridgeline SerACH (Cash)", 2273.03, cma) + categorize_expense("DIRECT DEPOSIT Ridgeline SerACH (Cash)", 2300.00, cma) == "Payroll" ) # Already-working variants must not regress. assert ( - categorize_expense("DIRECT DEPOSIT NORTHFIELD COLLPAYROLL (Cash)", 2431.51, cma) + categorize_expense("DIRECT DEPOSIT NORTHFIELD COLLPAYROLL (Cash)", 2400.00, cma) == "Payroll" ) # Sub-dollar bank verification deposits stay Exempt, not payroll. assert ( - categorize_expense("DIRECT DEPOSIT WELLS FARGO ACCTVERIFY", 0.29, cma) + categorize_expense("DIRECT DEPOSIT WELLS FARGO ACCTVERIFY", 0.25, cma) == "Exempt" ) + + +def test_sync_applies_household_merchant_rules(tmp_path) -> None: + """A merchant known only to merchant-rules.yaml reaches the database with + its household category; without the file it stays Uncategorized.""" + rules = tmp_path / "merchant-rules.yaml" + rules.write_text("Family Care:\n - little sprouts\n", encoding="utf-8") + database_url = f"sqlite:///{tmp_path / 'fo.db'}" + + def fake_dump(months: int, app_dir) -> dict: + return { + "accounts": [ + { + "id": "acct-1", + "name": "Rewards Card (5555)", + "org": {"name": "Any Bank"}, + "transactions": [ + { + "id": "t1", + "posted": 1755000000, + "amount": "-40.00", + "payee": "Little Sprouts", + "description": "LITTLE SPROUTS ANYTOWN TX", + } + ], + } + ] + } + + sync(database_url, dump_provider=fake_dump) + conn = sqlite3.connect(tmp_path / "fo.db") + assert conn.execute("SELECT category FROM bank_transactions").fetchone() == ( + "Uncategorized", + ) + sync(database_url, dump_provider=fake_dump, merchant_rules=rules) + assert conn.execute("SELECT category FROM bank_transactions").fetchone() == ( + "Family Care", + ) diff --git a/tests/python/test_snaptrade_db_sync.py b/tests/python/test_snaptrade_db_sync.py index dac22568..919f7443 100644 --- a/tests/python/test_snaptrade_db_sync.py +++ b/tests/python/test_snaptrade_db_sync.py @@ -119,8 +119,8 @@ def test_categorize_matches_and_prioritizes_transfers() -> None: """Merchant text maps to the right category; transfers win over merchant reads.""" from src.integrations.simplefin.categorize import categorize_expense as categorize - assert categorize("H-E-B #063 Pearland TX") == "Groceries" - assert categorize("CVS/PHARMACY # MANVEL TX") == "Health & Wellness" + assert categorize("H-E-B #063 Anytown TX") == "Groceries" + assert categorize("CVS/PHARMACY # ANYTOWN TX") == "Health & Wellness" assert categorize("Delta Airlines") == "Travel" # "American Express Travel" must read as Travel, not a bank/transfer assert categorize("American Express Travel") == "Travel" diff --git a/uv.lock b/uv.lock index bd3df1a4..fa25bcd6 100644 --- a/uv.lock +++ b/uv.lock @@ -739,7 +739,7 @@ wheels = [ [[package]] name = "family-office" -version = "2.3.2" +version = "2.3.3" source = { editable = "." } dependencies = [ { name = "anthropic" },