From b80015c0752076d07deb4400a0cf1eaefcb7e084 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 13:24:12 +0100 Subject: [PATCH 01/32] =?UTF-8?q?refactor(db):=20P0=20Postgres=20prep=20?= =?UTF-8?q?=E2=80=94=20RETURNING=20ids,=20ON=20CONFLICT=20upserts=20(SQLit?= =?UTF-8?q?e-safe)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First phase of the SQLite -> Supabase Postgres migration: dialect-neutral rewrites that run identically on SQLite >=3.35. - 17 cursor.lastrowid reads become INSERT ... RETURNING id fetches (users.db async stores x7, parses x3, zones curation x4, raids x3), each with its paired -- :name SQL block. - INSERT OR REPLACE x3 -> ON CONFLICT ... DO UPDATE (census character_aas/gear_sets, shared _meta upsert) — also fixes the delete-then-reinsert column-reset footgun. - INSERT OR IGNORE x9 -> ON CONFLICT DO NOTHING (zones x5, downloads, capped favorites, grant_role, erasure tombstone); rowcount semantics preserved where callers check them. - gitignore .mcp.json (local MCP config carries the Supabase token). No behaviour change; full suite green (2119). Co-Authored-By: Claude Fable 5 --- .gitignore | 4 ++++ backend/census/store.sql | 6 ++++-- backend/eq2db/_meta.sql | 3 ++- backend/eq2db/raids.py | 11 ++++++++--- backend/eq2db/raids.sql | 3 ++- backend/eq2db/zones.py | 9 +++++---- backend/eq2db/zones.sql | 26 +++++++++++++++++--------- backend/server/db/aa_plans.py | 3 ++- backend/server/db/aa_plans.sql | 3 ++- backend/server/db/attendance.py | 3 ++- backend/server/db/attendance.sql | 3 ++- backend/server/db/claims.py | 3 ++- backend/server/db/claims.sql | 3 ++- backend/server/db/downloads.sql | 3 ++- backend/server/db/erasure.py | 5 +++-- backend/server/db/favorites.sql | 5 +++-- backend/server/db/item_watch.py | 3 ++- backend/server/db/item_watch.sql | 3 ++- backend/server/db/raid_schedule.py | 3 ++- backend/server/db/raid_schedule.sql | 3 ++- backend/server/db/tokens.py | 3 ++- backend/server/db/tokens.sql | 3 ++- backend/server/db/users.py | 3 ++- backend/server/db/users.sql | 6 ++++-- backend/server/parses/db.py | 9 ++++++--- backend/server/parses/db.sql | 9 ++++++--- 26 files changed, 92 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index b17f53a9..92dae83b 100644 --- a/.gitignore +++ b/.gitignore @@ -346,3 +346,7 @@ data/census/census.db-shm backend/**/data/ scripts/dev/*_class_refs*/ scripts/dev/.epics_wikitext_cache.json + +# Local MCP server config (Claude Code) — carries the Supabase personal +# access token. Machine-local only, never committed. +.mcp.json diff --git a/backend/census/store.sql b/backend/census/store.sql index 68ece86e..eeebaec8 100644 --- a/backend/census/store.sql +++ b/backend/census/store.sql @@ -124,13 +124,15 @@ SELECT data_json, last_resolved_at FROM guilds WHERE name_lower=? AND world=?; SELECT data_json, last_resolved_at FROM character_aas WHERE name_lower = ? AND world = ?; -- :name upsert_character_aas -INSERT OR REPLACE INTO character_aas (name_lower, world, data_json, last_resolved_at) VALUES (?, ?, ?, ?); +INSERT INTO character_aas (name_lower, world, data_json, last_resolved_at) VALUES (?, ?, ?, ?) +ON CONFLICT(name_lower, world) DO UPDATE SET data_json = excluded.data_json, last_resolved_at = excluded.last_resolved_at; -- :name select_character_gear_sets SELECT data_json, last_resolved_at FROM character_gear_sets WHERE name_lower = ? AND world = ?; -- :name upsert_character_gear_sets -INSERT OR REPLACE INTO character_gear_sets (name_lower, world, data_json, last_resolved_at) VALUES (?, ?, ?, ?); +INSERT INTO character_gear_sets (name_lower, world, data_json, last_resolved_at) VALUES (?, ?, ?, ?) +ON CONFLICT(name_lower, world) DO UPDATE SET data_json = excluded.data_json, last_resolved_at = excluded.last_resolved_at; -- Name-prefix search over everything this server has ever seen — the -- store-first half of /characters/search and /guilds/search (the census diff --git a/backend/eq2db/_meta.sql b/backend/eq2db/_meta.sql index b4262b95..6caa548d 100644 --- a/backend/eq2db/_meta.sql +++ b/backend/eq2db/_meta.sql @@ -13,4 +13,5 @@ CREATE TABLE IF NOT EXISTS _meta ( SELECT value FROM _meta WHERE key = ? -- :name upsert -INSERT OR REPLACE INTO _meta (key, value) VALUES (?, ?) +INSERT INTO _meta (key, value) VALUES (?, ?) +ON CONFLICT(key) DO UPDATE SET value = excluded.value diff --git a/backend/eq2db/raids.py b/backend/eq2db/raids.py index 1edb748f..bc8cdd0f 100644 --- a/backend/eq2db/raids.py +++ b/backend/eq2db/raids.py @@ -317,7 +317,8 @@ def upsert_raid_encounter( actor if source != SOURCE_SCRAPE else None, ), ) - new_id = int(cur.lastrowid or 0) + _row = cur.fetchone() + new_id = int(_row[0]) if _row else 0 # First-ever revision row: before is NULL, after is the seeded content. if strategy_md is not None: conn.execute( @@ -494,11 +495,13 @@ def upsert_act_trigger( timer, timer_name, tabbed, cooldown_seconds, last_edited_at, last_edited_by ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + RETURNING id """, params, ) + _row = cur.fetchone() conn.commit() - return int(cur.lastrowid or 0) + return int(_row[0]) if _row else 0 conn.execute( """ @@ -595,11 +598,13 @@ def upsert_act_spell_timer( damage_type, control_effect, last_edited_at, last_edited_by ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + RETURNING id """, params, ) + _row = cur.fetchone() conn.commit() - return int(cur.lastrowid or 0) + return int(_row[0]) if _row else 0 conn.execute( """ diff --git a/backend/eq2db/raids.sql b/backend/eq2db/raids.sql index b3e0db29..da9c5cb8 100644 --- a/backend/eq2db/raids.sql +++ b/backend/eq2db/raids.sql @@ -240,7 +240,8 @@ INSERT INTO raid_encounters ( raid_zone_id, mob_name, mob_name_lower, position, strategy_md, wiki_url, source, last_synced_at, last_edited_at, last_edited_by -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +RETURNING id; -- Refresh sync timestamp + URL + position on a manually-edited row, leaving -- strategy_md alone — re-scrape doesn't overwrite curator edits. diff --git a/backend/eq2db/zones.py b/backend/eq2db/zones.py index f4e4749a..478bc0f2 100644 --- a/backend/eq2db/zones.py +++ b/backend/eq2db/zones.py @@ -431,7 +431,7 @@ def add_to_encounter( _SQL["insert_encounter_mob_primary"], (encounter_id, mob_name, mob_name.lower()), ) - new_id = cur.lastrowid + new_id = (cur.fetchone() or (None,))[0] conn.execute(_SQL["update_encounter_name"], (mob_name, encounter_id)) else: next_pos = conn.execute(_SQL["max_mob_position_for_encounter"], (encounter_id,)).fetchone()[0] @@ -439,7 +439,7 @@ def add_to_encounter( _SQL["insert_encounter_mob"], (encounter_id, mob_name, mob_name.lower(), next_pos), ) - new_id = cur.lastrowid + new_id = (cur.fetchone() or (None,))[0] row = conn.execute(_SQL["select_mob_by_id"], (new_id,)).fetchone() if make_primary and old_primary_name is not None: _mirror_primary_rename_in_raids_db(_encounter_zone_id(encounter_id, path), old_primary_name, mob_name, path) @@ -683,7 +683,8 @@ def replace_all_for_zone(cls, conn: sqlite3.Connection, zone_id: int, encounters enc.get("wiki_url"), ), ) - encounter_id = int(cur.lastrowid or 0) + _row = cur.fetchone() + encounter_id = int(_row[0]) if _row else 0 mobs = enc.get("mobs") or [] if not mobs: # Defensive: an encounter with no listed mobs gets one mob @@ -726,7 +727,7 @@ def add_to_zone( row = conn.execute(_SQL["max_encounter_position_for_zone"], (zone_id,)).fetchone() position = int(row["p"]) cur = conn.execute(_SQL["insert_encounter"], (zone_id, primary_mob, position, stage, wiki_url)) - enc_id = cur.lastrowid + enc_id = (cur.fetchone() or (None,))[0] conn.execute(_SQL["insert_encounter_mob_primary"], (enc_id, primary_mob, primary_mob.lower())) conn.commit() row = conn.execute(_SQL["select_encounter_by_id"], (enc_id,)).fetchone() diff --git a/backend/eq2db/zones.sql b/backend/eq2db/zones.sql index 51c3687e..d31e6d71 100644 --- a/backend/eq2db/zones.sql +++ b/backend/eq2db/zones.sql @@ -144,10 +144,11 @@ UPDATE zone_encounters -- Insert the parenthesised " (Zone)" form as an alias before stripping it -- from the canonical name, so old references resolve. -- :name normalise_paren_zone_to_alias -INSERT OR IGNORE INTO zone_aliases (alias, alias_lower, zone_id) +INSERT INTO zone_aliases (alias, alias_lower, zone_id) SELECT name, name_lower, id FROM zones - WHERE name LIKE '% (Zone)%'; + WHERE name LIKE '% (Zone)%' +ON CONFLICT DO NOTHING; -- Strip the " (Zone)" suffix from canonical zone names. -- :name normalise_strip_paren_zone @@ -288,7 +289,8 @@ DELETE FROM zone_types WHERE zone_id = ?; INSERT INTO zone_types (zone_id, type) VALUES (?, ?); -- :name insert_zone_type_or_ignore -INSERT OR IGNORE INTO zone_types (zone_id, type) VALUES (?, ?); +INSERT INTO zone_types (zone_id, type) VALUES (?, ?) +ON CONFLICT DO NOTHING; -- :name delete_zone_type DELETE FROM zone_types WHERE zone_id = ? AND type = ?; @@ -320,7 +322,8 @@ SELECT alias FROM zone_aliases WHERE zone_id = ? ORDER BY alias; DELETE FROM zone_encounters WHERE zone_id = ?; -- :name insert_encounter -INSERT INTO zone_encounters (zone_id, encounter_name, position, stage, wiki_url) VALUES (?, ?, ?, ?, ?); +INSERT INTO zone_encounters (zone_id, encounter_name, position, stage, wiki_url) VALUES (?, ?, ?, ?, ?) +RETURNING id; -- :name list_encounters_for_zone SELECT id, zone_id, encounter_name, position, stage, wiki_url @@ -358,10 +361,12 @@ SELECT COALESCE(MAX(position), 0) + 1 AS p FROM zone_encounters WHERE zone_id = -- --------------------------------------------------------------------------- -- :name insert_encounter_mob -INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (?, ?, ?, ?); +INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (?, ?, ?, ?) +RETURNING id; -- :name insert_encounter_mob_primary -INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (?, ?, ?, 0); +INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (?, ?, ?, 0) +RETURNING id; -- :name update_encounter_mob_primary_rename UPDATE zone_encounter_mobs SET mob_name = ?, mob_name_lower = ? @@ -455,7 +460,8 @@ WHERE z.expansion_short IS NOT NULL ORDER BY z.expansion_year DESC, z.expansion_short; -- :name insert_featured_raid_expansion -INSERT OR IGNORE INTO featured_raid_expansions (expansion_short) VALUES (?); +INSERT INTO featured_raid_expansions (expansion_short) VALUES (?) +ON CONFLICT DO NOTHING; -- :name remove_featured_raid_zones_in_expansion DELETE FROM featured_raid_zones @@ -491,7 +497,8 @@ JOIN zones z2 ON z2.id = f.zone_id WHERE z2.expansion_short = ? AND f.category IS NULL; -- :name insert_featured_raid_zone_uncategorised -INSERT OR IGNORE INTO featured_raid_zones (zone_id, position, category) VALUES (?, ?, NULL); +INSERT INTO featured_raid_zones (zone_id, position, category) VALUES (?, ?, NULL) +ON CONFLICT DO NOTHING; -- :name delete_featured_raid_zone_by_name DELETE FROM featured_raid_zones @@ -506,7 +513,8 @@ WHERE z.name_lower = ? AND z.expansion_short = ?; SELECT COALESCE(MAX(position), -1) FROM featured_raid_categories WHERE expansion_short = ?; -- :name insert_featured_raid_category_or_ignore -INSERT OR IGNORE INTO featured_raid_categories (expansion_short, name, position) VALUES (?, ?, ?); +INSERT INTO featured_raid_categories (expansion_short, name, position) VALUES (?, ?, ?) +ON CONFLICT DO NOTHING; -- :name insert_featured_raid_category INSERT INTO featured_raid_categories (expansion_short, name, position) VALUES (?, ?, ?); diff --git a/backend/server/db/aa_plans.py b/backend/server/db/aa_plans.py index 14c266cb..9a69b2c2 100644 --- a/backend/server/db/aa_plans.py +++ b/backend/server/db/aa_plans.py @@ -66,8 +66,9 @@ async def create_plan( _SQL["insert_plan"], (discord_id, world, character_name, name, xpac, allocations_json, slug), ) + row = await cur.fetchone() await db.commit() - plan_id = cur.lastrowid + plan_id = row[0] if row else 0 plan = await self.get_plan(int(plan_id or 0)) if plan is None: # pragma: no cover — insert+select on one path raise RuntimeError("aa_plan insert did not persist") diff --git a/backend/server/db/aa_plans.sql b/backend/server/db/aa_plans.sql index 3edea541..c39402c1 100644 --- a/backend/server/db/aa_plans.sql +++ b/backend/server/db/aa_plans.sql @@ -22,7 +22,8 @@ SELECT id, discord_id, world, character_name, name, xpac, allocations, share_slu -- :name insert_plan INSERT INTO aa_plans (discord_id, world, character_name, name, xpac, allocations, share_slug) -VALUES (?, ?, ?, ?, ?, ?, ?); +VALUES (?, ?, ?, ?, ?, ?, ?) +RETURNING id; -- :name update_plan UPDATE aa_plans diff --git a/backend/server/db/attendance.py b/backend/server/db/attendance.py index 32f388fc..f98e8a3e 100644 --- a/backend/server/db/attendance.py +++ b/backend/server/db/attendance.py @@ -144,7 +144,8 @@ async def apply_snapshot( json.dumps({discord_id: sent_at}), ), ) - session_id = cur.lastrowid + _row = await cur.fetchone() + session_id = _row[0] if _row else None for kind, members in (("raid", raid_members), ("online", online_guildies)): for m in members: diff --git a/backend/server/db/attendance.sql b/backend/server/db/attendance.sql index aa541cc7..3a4cfcaf 100644 --- a/backend/server/db/attendance.sql +++ b/backend/server/db/attendance.sql @@ -26,7 +26,8 @@ WHERE world = ? AND guild_name = ? AND session_day = ?; -- :name insert_session INSERT INTO attendance_sessions (world, guild_name, session_day, seq, started_at, ended_at, zones, scheduled, team_index, uploaders) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +RETURNING id; -- :name merge_session_window UPDATE attendance_sessions diff --git a/backend/server/db/claims.py b/backend/server/db/claims.py index 4d08f1cd..3f77e74f 100644 --- a/backend/server/db/claims.py +++ b/backend/server/db/claims.py @@ -89,7 +89,8 @@ async def submit_claim( _SQL["submit_claim"], (discord_id, character_name, world), ) - new_id = cur.lastrowid + _row = await cur.fetchone() + new_id = _row[0] if _row else None await db.commit() async with db.execute(_SQL["find_by_id"], (new_id,)) as cur2: row = await cur2.fetchone() diff --git a/backend/server/db/claims.sql b/backend/server/db/claims.sql index 558596e8..4e4bef35 100644 --- a/backend/server/db/claims.sql +++ b/backend/server/db/claims.sql @@ -16,7 +16,8 @@ WHERE discord_id = ? AND world = ? AND status = 'pending'; -- :name submit_claim INSERT INTO character_claims (discord_id, character_name, status, world) -VALUES (?, ?, 'pending', ?); +VALUES (?, ?, 'pending', ?) +RETURNING id; -- :name find_by_id SELECT * FROM character_claims WHERE id = ?; diff --git a/backend/server/db/downloads.sql b/backend/server/db/downloads.sql index ad155887..a518571e 100644 --- a/backend/server/db/downloads.sql +++ b/backend/server/db/downloads.sql @@ -4,7 +4,8 @@ -- user, so the public count is distinct-downloaders and can't be inflated by -- one person re-clicking. rowcount 0 ⇒ this user already recorded this slug. -- :name insert_download -INSERT OR IGNORE INTO download_events (discord_id, slug) VALUES (?, ?); +INSERT INTO download_events (discord_id, slug) VALUES (?, ?) +ON CONFLICT DO NOTHING; -- :name count_for_slug SELECT COUNT(*) FROM download_events WHERE slug = ?; diff --git a/backend/server/db/erasure.py b/backend/server/db/erasure.py index c54d2f82..741a43d8 100644 --- a/backend/server/db/erasure.py +++ b/backend/server/db/erasure.py @@ -103,8 +103,9 @@ def _table_exists(conn: sqlite3.Connection, table: str) -> bool: def _ensure_tombstone(conn: sqlite3.Connection, now: int) -> None: conn.execute( - "INSERT OR IGNORE INTO users (discord_id, discord_name, discord_username, avatar, first_seen, last_seen, " - "access_status) VALUES (?, 'Deleted user', 'deleted', NULL, ?, ?, 'denied')", + "INSERT INTO users (discord_id, discord_name, discord_username, avatar, first_seen, last_seen, " + "access_status) VALUES (?, 'Deleted user', 'deleted', NULL, ?, ?, 'denied') " + "ON CONFLICT(discord_id) DO NOTHING", (DELETED_USER_ID, now, now), ) diff --git a/backend/server/db/favorites.sql b/backend/server/db/favorites.sql index 1a36e6b7..d21f75df 100644 --- a/backend/server/db/favorites.sql +++ b/backend/server/db/favorites.sql @@ -5,9 +5,10 @@ -- already existed (OR IGNORE) or the cap was hit — callers disambiguate with -- select_is_favorited. -- :name insert_favorite_capped -INSERT OR IGNORE INTO character_favorites (discord_id, character_name, world) +INSERT INTO character_favorites (discord_id, character_name, world) SELECT ?, ?, ? -WHERE (SELECT COUNT(*) FROM character_favorites WHERE discord_id = ? AND world = ?) < ?; +WHERE (SELECT COUNT(*) FROM character_favorites WHERE discord_id = ? AND world = ?) < ? +ON CONFLICT DO NOTHING; -- :name delete_favorite DELETE FROM character_favorites WHERE discord_id = ? AND character_name = ? AND world = ?; diff --git a/backend/server/db/item_watch.py b/backend/server/db/item_watch.py index 5dd3add5..29b73eb3 100644 --- a/backend/server/db/item_watch.py +++ b/backend/server/db/item_watch.py @@ -49,7 +49,8 @@ async def add_item_watch( if "UNIQUE" in str(exc): raise ValueError(f"'{item_name}' is already being watched for {character_name}.") from exc raise - new_id = cur.lastrowid + _row = await cur.fetchone() + new_id = _row[0] if _row else None await db.commit() async with db.execute(_SQL["find_by_id"], (new_id,)) as cur2: row = await cur2.fetchone() diff --git a/backend/server/db/item_watch.sql b/backend/server/db/item_watch.sql index dcc71175..bd96c6ec 100644 --- a/backend/server/db/item_watch.sql +++ b/backend/server/db/item_watch.sql @@ -3,7 +3,8 @@ -- :name add_watch INSERT INTO item_watch (world, guild_name, character_name, item_id, item_name, added_by, added_by_name) -VALUES (?, ?, ?, ?, ?, ?, ?); +VALUES (?, ?, ?, ?, ?, ?, ?) +RETURNING id; -- :name find_by_id SELECT * FROM item_watch WHERE id = ?; diff --git a/backend/server/db/raid_schedule.py b/backend/server/db/raid_schedule.py index 17f5b662..2e9a1c87 100644 --- a/backend/server/db/raid_schedule.py +++ b/backend/server/db/raid_schedule.py @@ -73,7 +73,8 @@ async def replace_schedule( updated_by, ), ) - team_id = cur.lastrowid + _row = await cur.fetchone() + team_id = _row[0] if _row else None for slot_index, raid in enumerate(team.get("raids", [])): await db.execute( _SQL["insert_slot"], diff --git a/backend/server/db/raid_schedule.sql b/backend/server/db/raid_schedule.sql index 89860413..4e5a6997 100644 --- a/backend/server/db/raid_schedule.sql +++ b/backend/server/db/raid_schedule.sql @@ -21,7 +21,8 @@ DELETE FROM raid_teams WHERE world = ? AND guild_name = ?; -- :name insert_team INSERT INTO raid_teams (world, guild_name, team_index, name, primary_tz, twitch_login, updated_by) -VALUES (?, ?, ?, ?, ?, ?, ?); +VALUES (?, ?, ?, ?, ?, ?, ?) +RETURNING id; -- :name insert_slot INSERT INTO raid_slots (team_id, slot_index, days, start_min, end_min, label) diff --git a/backend/server/db/tokens.py b/backend/server/db/tokens.py index a4cadf3d..0737e72a 100644 --- a/backend/server/db/tokens.py +++ b/backend/server/db/tokens.py @@ -70,7 +70,8 @@ async def mint_api_token( _SQL["mint_token"], (user_id, name, h, prefix), ) - new_id = cur.lastrowid + _row = await cur.fetchone() + new_id = _row[0] if _row else None await db.commit() async with db.execute(_SQL["find_by_id"], (new_id,)) as cur2: row = await cur2.fetchone() diff --git a/backend/server/db/tokens.sql b/backend/server/db/tokens.sql index 6c3f9c0c..a64fc5e3 100644 --- a/backend/server/db/tokens.sql +++ b/backend/server/db/tokens.sql @@ -3,7 +3,8 @@ -- :name mint_token INSERT INTO api_tokens (user_id, name, token_hash, token_prefix) -VALUES (?, ?, ?, ?); +VALUES (?, ?, ?, ?) +RETURNING id; -- :name find_by_id SELECT * FROM api_tokens WHERE id = ?; diff --git a/backend/server/db/users.py b/backend/server/db/users.py index 6cc8ec0e..b024f63e 100644 --- a/backend/server/db/users.py +++ b/backend/server/db/users.py @@ -194,8 +194,9 @@ async def create_role_request( _SQL["create_role_request"], (discord_id, role, user_note), ) + row = await cur.fetchone() await db.commit() - return int(cur.lastrowid or 0) + return int(row[0]) if row else 0 async def list_role_requests( self, diff --git a/backend/server/db/users.sql b/backend/server/db/users.sql index b7fde980..300d911e 100644 --- a/backend/server/db/users.sql +++ b/backend/server/db/users.sql @@ -50,7 +50,8 @@ UPDATE users SET access_status = ? WHERE discord_id = ?; -- --------------------------------------------------------------------------- -- :name grant_role -INSERT OR IGNORE INTO user_roles (discord_id, role, granted_by) VALUES (?, ?, ?); +INSERT INTO user_roles (discord_id, role, granted_by) VALUES (?, ?, ?) +ON CONFLICT DO NOTHING; -- :name revoke_role DELETE FROM user_roles WHERE discord_id = ? AND role = ?; @@ -69,7 +70,8 @@ SELECT discord_id, role FROM user_roles ORDER BY discord_id, role; -- --------------------------------------------------------------------------- -- :name create_role_request -INSERT INTO role_requests (discord_id, role, user_note) VALUES (?, ?, ?); +INSERT INTO role_requests (discord_id, role, user_note) VALUES (?, ?, ?) +RETURNING id; -- {where_sql} = "WHERE …" or "" composed by Python build_where helper. -- {order_sql} = "ORDER BY …" composed by Python (varies by status filter). diff --git a/backend/server/parses/db.py b/backend/server/parses/db.py index 2d8cb9d0..004d9388 100644 --- a/backend/server/parses/db.py +++ b/backend/server/parses/db.py @@ -255,7 +255,8 @@ def insert_encounter( guild_name=guild_name, ), ) - return int(cur.lastrowid or 0) + row = cur.fetchone() + return int(row[0]) if row else 0 @staticmethod def insert_combatants_bulk( @@ -274,7 +275,8 @@ def insert_combatants_bulk( _SQL["insert_combatant"], c.as_db_params(encounter_id=encounter_id, snapshot=snap), ) - name_to_id[c.name] = int(cur.lastrowid or 0) + row = cur.fetchone() + name_to_id[c.name] = int(row[0]) if row else 0 return name_to_id @staticmethod @@ -716,7 +718,8 @@ def insert_tamper_report( payload_json, ), ) - return int(cur.lastrowid or 0) + row = cur.fetchone() + return int(row[0]) if row else 0 @staticmethod def list_tamper_reports( diff --git a/backend/server/parses/db.sql b/backend/server/parses/db.sql index 8702e2d8..6633a8fd 100644 --- a/backend/server/parses/db.sql +++ b/backend/server/parses/db.sql @@ -328,7 +328,8 @@ INSERT INTO encounters ( :started_at, :ended_at, :duration_s, :total_damage, :encdps, :kills, :deaths, :success_level, :source_dsn, :uploaded_by, :guild_name, :ingested_at -); +) +RETURNING id; -- Named :param placeholders matched against Combatant.as_db_params(). -- :name insert_combatant @@ -356,7 +357,8 @@ INSERT INTO combatants ( :to_hit, :crit_dam_perc, :crit_heal_perc, :crit_types, :threat_str, :threat_delta, :level, :guild_name, :cls, :ilvl -); +) +RETURNING id; -- :name update_combatant_snapshot UPDATE combatants SET level = ?, guild_name = ?, cls = ?, ilvl = ? @@ -517,7 +519,8 @@ INSERT INTO tamper_reports ( ?, ?, ?, ?, ?, ?, ? -); +) +RETURNING id; -- :name list_tamper_reports -- {where} composed in Python (filters: world / reason / pending|ack|all) From 50584ef2ae73052e07e33c3b28405131869092cc Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 13:28:15 +0100 Subject: [PATCH 02/32] feat(metrics): parse-detail age-bucket analytics; pg connection seam + deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse_detail_views_total{age_bucket} counts detail-page opens by the encounter's age (<7d / 7-30d / 30-90d / >90d) — the data that will decide whether attack/damage-type detail older than N days can take a rolling retention window on Postgres (user decision: Supabase Pro now, revisit pruning once this histogram has months of data). Also lands backend/pg.py (DSN resolution DATABASE_URL -> SUPABASE_DB_URL -> POSTGRES_CONNECTION_STRING; async+sync psycopg pools opened in the app lifespan; pool-if-open-else-direct checkout contextmanagers) and the psycopg[binary]/psycopg-pool dependencies — the P1 connection seam. Co-Authored-By: Claude Fable 5 --- backend/pg.py | 96 +++++++++++++++++++++++++++++++ backend/server/api/parses/list.py | 7 +++ backend/server/metrics.py | 27 +++++++++ pyproject.toml | 2 + tests/server/test_metrics.py | 16 ++++++ uv.lock | 73 +++++++++++++++++++++++ 6 files changed, 221 insertions(+) create mode 100644 backend/pg.py diff --git a/backend/pg.py b/backend/pg.py new file mode 100644 index 00000000..c4141127 --- /dev/null +++ b/backend/pg.py @@ -0,0 +1,96 @@ +"""Shared Postgres connection plumbing for the migrated store families. + +One database, five schemas (users / parses / census / zones / raids — +see db/migrations/). Stores select their schema via ``SET search_path`` +at connection checkout, so every query stays unqualified. + +Pool-if-open-else-direct: the app lifespan opens one async + one sync +pool (`open_pools`) and every checkout goes through them; when no pool +is open — pytest (fresh event loop per test), scripts, or the bot's +startup racing the web lifespan — the same contextmanagers fall back to +a short-lived direct connection. Tests therefore never share a +loop-bound pool, and nothing outside the lifespan needs pool wiring. + +Production DSN is the Supabase Supavisor SESSION pooler (:5432, +sslmode=require). Transaction mode (:6543) would break psycopg's +prepared statements and the per-checkout search_path — don't use it. +""" + +from __future__ import annotations + +import os +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager, contextmanager + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, ConnectionPool + +#: Accepted DSN env names, in precedence order. DATABASE_URL is the +#: canonical name (Railway); the others are historical/dashboard-copied. +_DSN_VARS = ("DATABASE_URL", "SUPABASE_DB_URL", "POSTGRES_CONNECTION_STRING") + + +def dsn() -> str: + for var in _DSN_VARS: + value = os.getenv(var) + if value: + return value + raise RuntimeError(f"No Postgres DSN configured — set one of {', '.join(_DSN_VARS)}.") + + +_async_pool: AsyncConnectionPool | None = None +_sync_pool: ConnectionPool | None = None + + +async def open_pools() -> None: + """Open the shared pools — called once from the app lifespan startup. + Small on purpose: WEB_CONCURRENCY=1 and web+bot share one process, so + ~10 server connections total stays well inside Supabase's session- + pooler limits.""" + global _async_pool, _sync_pool + _async_pool = AsyncConnectionPool(dsn(), min_size=1, max_size=5, open=False, kwargs={"row_factory": dict_row}) + await _async_pool.open() + _sync_pool = ConnectionPool(dsn(), min_size=1, max_size=5, kwargs={"row_factory": dict_row}) + + +async def close_pools() -> None: + """Lifespan shutdown. Safe to call when pools never opened.""" + global _async_pool, _sync_pool + if _async_pool is not None: + await _async_pool.close() + _async_pool = None + if _sync_pool is not None: + _sync_pool.close() + _sync_pool = None + + +@asynccontextmanager +async def aconnection() -> AsyncIterator[psycopg.AsyncConnection]: + """Async checkout: pooled when the lifespan opened pools, else a + short-lived direct connection (tests / scripts / pre-lifespan bot). + psycopg commits on clean ``async with`` exit and rolls back on + exception — explicit commits inside remain harmless.""" + if _async_pool is not None: + async with _async_pool.connection() as conn: + yield conn + else: + async with await psycopg.AsyncConnection.connect(dsn(), row_factory=dict_row) as conn: + yield conn + + +@contextmanager +def connection() -> Iterator[psycopg.Connection]: + """Sync twin of :func:`aconnection`.""" + if _sync_pool is not None: + with _sync_pool.connection() as conn: + yield conn + else: + with psycopg.connect(dsn(), row_factory=dict_row) as conn: + yield conn + + +def search_path_sql(schema: str) -> psycopg.sql.Composed: + """The one statement stores run at checkout — composed so a schema + name can never inject (test scratch schemas are generated names).""" + return psycopg.sql.SQL("SET search_path TO {}, public").format(psycopg.sql.Identifier(schema)) diff --git a/backend/server/api/parses/list.py b/backend/server/api/parses/list.py index e8a87f98..92979438 100644 --- a/backend/server/api/parses/list.py +++ b/backend/server/api/parses/list.py @@ -48,6 +48,7 @@ ) from backend.server.core.executor import run_sync from backend.server.limiter import limiter +from backend.server.metrics import PARSE_DETAIL_VIEWS, parse_age_bucket from backend.server.parses.db import store as parses_db from backend.server.parses.pet_detection import classify_combatants from backend.server.server_context import current_world @@ -761,6 +762,12 @@ async def get_parse( if enc is None: raise HTTPException(status_code=404, detail="Parse not found") + # Access analytics: how old is the parse people actually open? Feeds + # the future attack/damage-type retention decision (see metrics.py). + import time as _time # noqa: PLC0415 + + PARSE_DETAIL_VIEWS.labels(parse_age_bucket(enc.get("started_at"), int(_time.time()))).inc() + # encDPS percentile colouring: rank each combatant's encDPS against their # class's best for this boss (class leader = 100%), and flag the all-class # best with a star. Empty for non-boss encounters (no matching kills). diff --git a/backend/server/metrics.py b/backend/server/metrics.py index 87a77011..15338263 100644 --- a/backend/server/metrics.py +++ b/backend/server/metrics.py @@ -109,6 +109,33 @@ def collect(self): # type: ignore[override] # ── Cache metrics ───────────────────────────────────────────────────────────── # Labels: cache = character | guild | claim +# How often parse DETAIL pages are opened, bucketed by the encounter's +# age at view time. Gathered to decide whether the per-ability breakdown +# tables (attack_types/damage_types) can take a rolling retention window +# once parses live on Postgres — if the >90d bucket stays ~0, old detail +# can be pruned and the DB fits a smaller tier. Buckets: <7d, 7-30d, +# 30-90d, >90d. +PARSE_DETAIL_VIEWS = Counter( + "parse_detail_views_total", + "Parse detail page loads by encounter age at view time", + ["age_bucket"], +) + + +def parse_age_bucket(started_at: int | None, now: int) -> str: + """Age bucket label for PARSE_DETAIL_VIEWS.""" + if not started_at: + return "unknown" + days = max(0, now - int(started_at)) / 86400 + if days < 7: + return "<7d" + if days < 30: + return "7-30d" + if days < 90: + return "30-90d" + return ">90d" + + CACHE_HITS = Counter("cache_hits_total", "Fresh cache hits", ["cache"]) CACHE_MISSES = Counter("cache_misses_total", "Cache misses (not found or expired)", ["cache"]) CACHE_STALE = Counter("cache_stale_total", "Stale hits that fired bg refresh", ["cache"]) diff --git a/pyproject.toml b/pyproject.toml index dd3a62e9..24322558 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ dependencies = [ # so we don't hand-curate a slur list in the repo. See # backend/server/core/text_moderation.py. "better-profanity>=0.7.0", + "psycopg[binary]>=3.2", + "psycopg-pool>=3.2", ] [dependency-groups] diff --git a/tests/server/test_metrics.py b/tests/server/test_metrics.py index b743a2e7..3ba9576d 100644 --- a/tests/server/test_metrics.py +++ b/tests/server/test_metrics.py @@ -75,3 +75,19 @@ def test_record_user_seen_prunes_stale_entries(monkeypatch): metrics._user_last_seen[str(i)] = now - 100_000 # all stale metrics.record_user_seen("fresh") assert metrics._user_last_seen == {"fresh": now} + + +def test_parse_age_bucket_boundaries(): + """Bucket labels for the parse-detail access analytics — the data that + will decide whether old attack/damage-type detail can be pruned.""" + now = 1_800_000_000 + day = 86_400 + assert metrics.parse_age_bucket(None, now) == "unknown" + assert metrics.parse_age_bucket(0, now) == "unknown" # missing timestamp + assert metrics.parse_age_bucket(now, now) == "<7d" + assert metrics.parse_age_bucket(now - 6 * day, now) == "<7d" + assert metrics.parse_age_bucket(now - 7 * day, now) == "7-30d" + assert metrics.parse_age_bucket(now - 29 * day, now) == "7-30d" + assert metrics.parse_age_bucket(now - 30 * day, now) == "30-90d" + assert metrics.parse_age_bucket(now - 90 * day, now) == ">90d" + assert metrics.parse_age_bucket(now + 500, now) == "<7d" # clock skew clamps diff --git a/uv.lock b/uv.lock index fc456642..6bcfdac5 100644 --- a/uv.lock +++ b/uv.lock @@ -297,6 +297,8 @@ dependencies = [ { name = "mwparserfromhell" }, { name = "pillow" }, { name = "prometheus-client" }, + { name = "psycopg", extra = ["binary"] }, + { name = "psycopg-pool" }, { name = "python-dotenv" }, { name = "python-multipart" }, { name = "slowapi" }, @@ -325,6 +327,8 @@ requires-dist = [ { name = "mwparserfromhell", specifier = ">=0.6.0" }, { name = "pillow", specifier = ">=10.0.0" }, { name = "prometheus-client", specifier = ">=0.26.0" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, + { name = "psycopg-pool", specifier = ">=3.2" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "python-multipart", specifier = ">=0.0.9" }, { name = "slowapi", specifier = ">=0.1.9" }, @@ -808,6 +812,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b9/60711317c284a442511644ea7185b56ebe627606d6741e732cd16108c47b/psycopg_binary-3.3.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba", size = 4720512, upload-time = "2026-09-18T13:20:29.278Z" }, + { url = "https://files.pythonhosted.org/packages/63/da/28befc84454cbc6374550de7746f591f8fe1b6165c1fce249652cc8291c4/psycopg_binary-3.3.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4", size = 4782318, upload-time = "2026-09-18T13:20:35.401Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8a/0d21c2c833cdc0d4244c77e858e0ed37fa2abec2623be4fd686f617109ce/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475", size = 5567460, upload-time = "2026-09-18T13:20:41.902Z" }, + { url = "https://files.pythonhosted.org/packages/49/6d/7692d0d4e656b6cc9868d8acc2e3b42f17a0db4a625400a6d093cb0533a1/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5", size = 5246902, upload-time = "2026-09-18T13:20:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c1/b8a1f18fb1b7558a17f57f7cb3fc8bc93189feea2958925950b3acb15743/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a", size = 6847192, upload-time = "2026-09-18T13:20:56.874Z" }, + { url = "https://files.pythonhosted.org/packages/a5/76/404f33519167c65cca88ec4998776f1dbebccc301ee977f0e62c47fb0826/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638", size = 5079573, upload-time = "2026-09-18T13:21:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d9/79e8fbc8f37262a415f3550f0bcc5f98037442bf3d12ef6cbae2056655ae/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7", size = 4613633, upload-time = "2026-09-18T13:21:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/d4/47/96225db74be7d2ce04b3a58678b53cda610225055edf5faa775c9f501d8b/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e", size = 4293375, upload-time = "2026-09-18T13:21:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d2/18e9c779a5efd565250329adaf529ecc2b8b2ed5be5cb0f6ccee208cbfd9/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6", size = 4019883, upload-time = "2026-09-18T13:21:21.587Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/0cc654afc6c2cda982767f5679d3646b30b1ec86545bdaa9402202d6776c/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781", size = 4332607, upload-time = "2026-09-18T13:21:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3e/0a753a74fbd7aef120f286c016e09d3cc3f1daf7688f4a145d27281260b2/psycopg_binary-3.3.6-cp314-cp314-win_amd64.whl", hash = "sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840", size = 3755671, upload-time = "2026-09-18T13:21:33.855Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b1/a372b9c02aea50148e71c9853e19efca8fa5ae2010a8e27243b9b8f790c0/psycopg_binary-3.3.6-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c", size = 4719571, upload-time = "2026-09-18T13:21:41.437Z" }, + { url = "https://files.pythonhosted.org/packages/65/7c/811e3828c6b82e2f10c6c9cdd963cfc66f3e024026e5a69ac18530bad984/psycopg_binary-3.3.6-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a", size = 4781230, upload-time = "2026-09-18T13:21:49.516Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/9a784eed813ea9e97c294af3ead63d02b7b203502c66380336c50065e441/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc", size = 5566111, upload-time = "2026-09-18T13:21:58.089Z" }, + { url = "https://files.pythonhosted.org/packages/68/16/47194e002007c27337b11e49bf459c4b19727463f9aff2e1a90917bcc806/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e", size = 5249963, upload-time = "2026-09-18T13:22:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/5dcf9f310b11f0675cd860c6b2c70f58ce61798a3ee3f6f962b53fa358ca/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312", size = 6847925, upload-time = "2026-09-18T13:22:13.088Z" }, + { url = "https://files.pythonhosted.org/packages/f3/06/1957a06dc22963c418c27b284929579de84f29c37ad1abe6dc6ee9e8cf25/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1", size = 5087720, upload-time = "2026-09-18T13:22:17.959Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/ac07d042bae99b57bf123bb473632f29af544008094da0ffd285ab8011e2/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10", size = 4613412, upload-time = "2026-09-18T13:22:26.719Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b1/019156fbeafcefb4cccc9d109de4699493bceb8313c7545c8349e089dfbc/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2", size = 4292618, upload-time = "2026-09-18T13:22:33.042Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0f/62113dc6b1df65983a1f2fc816c04b1edfa22f2ae9d4abee74ed267f4a96/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8", size = 4027121, upload-time = "2026-09-18T13:22:38.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d5/cf0cbd1ea5a7d8167fe2c6953efde19101f7b193bd61a23e6d622ad6854c/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e", size = 4336388, upload-time = "2026-09-18T13:22:45.576Z" }, + { url = "https://files.pythonhosted.org/packages/98/33/e2a5b36edf8aa422f6fa4b894756eb33dc93b36df5f65121280bb8b929c4/psycopg_binary-3.3.6-cp315-cp315-win_amd64.whl", hash = "sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b", size = 3756154, upload-time = "2026-09-18T13:22:51.283Z" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" From 5690e74ca03cbcb7d44109c249e5301bbfe3acde Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 13:37:57 +0100 Subject: [PATCH 03/32] feat(pg): migration runner + reviewed users-family schema (0001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend/pg_migrate.py: ordered db/migrations/*.sql applier with a public.schema_migrations ledger, advisory-locked, per-file transactions; retarget() rewrites a family file's two-line header at a scratch schema — the test-isolation mechanism (validated: users_s1 provisions all 25 tables from the same file). db/migrations/0001_users.sql: the users.db schema translated AND reviewed, not mechanically ported — identity bigints, epoch-bigint defaults, deferrable FKs to users(discord_id) (erasure becomes one SET CONSTRAINTS ALL DEFERRED transaction), attendance zones/uploaders to jsonb (exact key-exists erasure probe verified), item_watch index re-scoped to (world, guild_name), redundant token-hash index dropped, idx_claims_primary made partial, idempotent seeds section (re-run by the scratch-schema reset). Verified end-to-end against local PostgreSQL 17. Co-Authored-By: Claude Fable 5 --- backend/pg.py | 14 +- backend/pg_migrate.py | 106 ++++++++++ db/migrations/0001_users.sql | 397 +++++++++++++++++++++++++++++++++++ 3 files changed, 511 insertions(+), 6 deletions(-) create mode 100644 backend/pg_migrate.py create mode 100644 db/migrations/0001_users.sql diff --git a/backend/pg.py b/backend/pg.py index c4141127..2b29e00d 100644 --- a/backend/pg.py +++ b/backend/pg.py @@ -21,8 +21,10 @@ import os from collections.abc import AsyncIterator, Iterator from contextlib import asynccontextmanager, contextmanager +from typing import Any import psycopg +from psycopg import sql as pgsql from psycopg.rows import dict_row from psycopg_pool import AsyncConnectionPool, ConnectionPool @@ -66,7 +68,7 @@ async def close_pools() -> None: @asynccontextmanager -async def aconnection() -> AsyncIterator[psycopg.AsyncConnection]: +async def aconnection() -> AsyncIterator[psycopg.AsyncConnection[Any]]: """Async checkout: pooled when the lifespan opened pools, else a short-lived direct connection (tests / scripts / pre-lifespan bot). psycopg commits on clean ``async with`` exit and rolls back on @@ -75,22 +77,22 @@ async def aconnection() -> AsyncIterator[psycopg.AsyncConnection]: async with _async_pool.connection() as conn: yield conn else: - async with await psycopg.AsyncConnection.connect(dsn(), row_factory=dict_row) as conn: + async with await psycopg.AsyncConnection.connect(dsn(), row_factory=dict_row) as conn: # type: ignore[arg-type] yield conn @contextmanager -def connection() -> Iterator[psycopg.Connection]: +def connection() -> Iterator[psycopg.Connection[Any]]: """Sync twin of :func:`aconnection`.""" if _sync_pool is not None: with _sync_pool.connection() as conn: yield conn else: - with psycopg.connect(dsn(), row_factory=dict_row) as conn: + with psycopg.connect(dsn(), row_factory=dict_row) as conn: # type: ignore[arg-type] yield conn -def search_path_sql(schema: str) -> psycopg.sql.Composed: +def search_path_sql(schema: str) -> pgsql.Composed: """The one statement stores run at checkout — composed so a schema name can never inject (test scratch schemas are generated names).""" - return psycopg.sql.SQL("SET search_path TO {}, public").format(psycopg.sql.Identifier(schema)) + return pgsql.SQL("SET search_path TO {}, public").format(pgsql.Identifier(schema)) diff --git a/backend/pg_migrate.py b/backend/pg_migrate.py new file mode 100644 index 00000000..69fde982 --- /dev/null +++ b/backend/pg_migrate.py @@ -0,0 +1,106 @@ +"""Tiny raw-SQL migration runner — the single applier for local dev, +tests, and the Supabase project (no Supabase CLI in the apply path). + +Files live in db/migrations/NNNN_.sql and run in filename order, +each inside its own transaction, recorded in public.schema_migrations. +A pg_advisory_lock serialises concurrent runs (overlapping Railway +containers during a deploy). + +FAMILY-FILE CONVENTION (meta-test enforced): a migration that targets +one store family starts with exactly these two lines:: + + create schema if not exists ; + set search_path to , public; + +followed by UNQUALIFIED DDL. The test fixtures retarget the same file at +scratch schemas (users_s1, ...) by substituting those two lines — one +source of truth for prod, dev, and per-test schemas. A file whose first +line is ``-- global`` is exempt (roles/grants). +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +import psycopg + +from backend import pg + +_log = logging.getLogger(__name__) + +MIGRATIONS_DIR = Path(__file__).resolve().parents[1] / "db" / "migrations" + +#: One stable lock id for the whole app's migration critical section. +_ADVISORY_LOCK_ID = 0x_E92_1E71 # arbitrary constant, spells-ish "eq2lexi" + +_LEDGER_DDL = """ +CREATE TABLE IF NOT EXISTS public.schema_migrations ( + filename text PRIMARY KEY, + applied_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) +) +""" + + +def migration_files(directory: Path | None = None) -> list[Path]: + d = directory or MIGRATIONS_DIR + return sorted(p for p in d.glob("*.sql") if p.is_file()) + + +def retarget(sql_text: str, schema: str) -> str: + """Rewrite a FAMILY migration's two-line header at ``schema`` — the + test scratch-schema mechanism. Raises on files that don't follow the + convention (the meta-test keeps them honest).""" + lines = sql_text.splitlines() + if lines and lines[0].strip() == "-- global": + raise ValueError("global migrations cannot be retargeted") + if ( + len(lines) < 2 + or not lines[0].strip().lower().startswith("create schema if not exists ") + or not lines[1].strip().lower().startswith("set search_path to ") + ): + raise ValueError("migration does not follow the two-line family header convention") + head = [ + f"create schema if not exists {schema};", + f"set search_path to {schema}, public;", + ] + return "\n".join(head + lines[2:]) + + +def apply_migrations(conn: psycopg.Connection[Any], *, directory: Path | None = None) -> list[str]: + """Apply every unapplied migration file in order. Returns the applied + filenames. The connection should be autocommit=False (default); each + file commits individually so a failure leaves earlier files applied + and recorded.""" + applied: list[str] = [] + conn.execute(_LEDGER_DDL) + conn.execute("SELECT pg_advisory_lock(%s)", (_ADVISORY_LOCK_ID,)) + try: + conn.commit() + done = {r["filename"] for r in conn.execute("SELECT filename FROM public.schema_migrations").fetchall()} + for path in migration_files(directory): + if path.name in done: + continue + _log.info("[pg-migrate] applying %s", path.name) + conn.execute(path.read_text(encoding="utf-8")) # type: ignore[arg-type] + conn.execute( + "INSERT INTO public.schema_migrations (filename) VALUES (%s)", + (path.name,), + ) + conn.commit() + applied.append(path.name) + except Exception: + conn.rollback() + raise + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (_ADVISORY_LOCK_ID,)) + conn.commit() + return applied + + +def run(*, directory: Path | None = None) -> list[str]: + """Open a connection (pool-less safe) and apply. The app lifespan and + conftest both call this.""" + with pg.connection() as conn: + return apply_migrations(conn, directory=directory) diff --git a/db/migrations/0001_users.sql b/db/migrations/0001_users.sql new file mode 100644 index 00000000..a72ef34c --- /dev/null +++ b/db/migrations/0001_users.sql @@ -0,0 +1,397 @@ +create schema if not exists users; +set search_path to users, public; + +-- ============================================================================ +-- users family — translated AND REVIEWED from backend/server/db/schema.sql +-- (the SQLite users.db), not a mechanical port. Type policy: identity +-- bigints; unix-epoch bigints (NOT timestamptz — Python arithmetic + the +-- JSON contract use raw ints); 0/1 flags stay integers (frontend contract); +-- ISO day strings stay text. Review decisions are annotated per table; +-- contract-affecting ideas were escalated instead of applied. +-- +-- FK policy: every reference to users(discord_id) is DEFERRABLE INITIALLY +-- IMMEDIATE so the erasure sweep runs as ONE transaction under +-- SET CONSTRAINTS ALL DEFERRED (replaces SQLite's PRAGMA foreign_keys=OFF). +-- Unlike SQLite (where users.db never enabled the pragma), these are +-- actually enforced. +-- ============================================================================ + +CREATE TABLE users ( + discord_id text PRIMARY KEY, + discord_name text NOT NULL, + discord_username text, + avatar text, + first_seen bigint NOT NULL DEFAULT floor(extract(epoch from now())), + last_seen bigint NOT NULL DEFAULT floor(extract(epoch from now())), + access_status text NOT NULL DEFAULT 'pending' +); + +-- REVIEW: considered CHECK constraints on the status enums (here and on +-- claims/requests/availability). Rejected: the routes validate them, the +-- value sets grow as data-not-schema, and ingest paths must never start +-- rejecting values SQLite accepted. + +CREATE TABLE character_claims ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + discord_id text NOT NULL REFERENCES users(discord_id) DEFERRABLE INITIALLY IMMEDIATE, + character_name text NOT NULL, + status text NOT NULL DEFAULT 'pending', + requested_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + reviewed_at bigint, + reviewed_by text, + note text, + world text NOT NULL DEFAULT 'Varsoon', + is_primary integer NOT NULL DEFAULT 0 +); + +CREATE INDEX idx_claims_discord ON character_claims (discord_id); +CREATE INDEX idx_claims_status ON character_claims (status); +CREATE INDEX idx_claims_world ON character_claims (world); +-- Ported from migrations.sql (post-ALTER home no longer needed — the +-- two-phase schema/migrations split is dead on Postgres). +CREATE INDEX idx_claims_primary ON character_claims (world, is_primary) WHERE is_primary = 1; + +CREATE TABLE item_watch ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + world text NOT NULL DEFAULT 'Varsoon', + guild_name text NOT NULL, + character_name text NOT NULL, + item_id bigint NOT NULL, -- census item ids exceed int4 + item_name text NOT NULL, + added_by text NOT NULL REFERENCES users(discord_id) DEFERRABLE INITIALLY IMMEDIATE, + added_by_name text NOT NULL, + added_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + first_seen_at bigint, -- first time we saw them wearing it (NULL = never) + last_seen_at bigint, -- most recent check where they had it equipped + last_checked_at bigint, -- most recent check (any result) + UNIQUE (world, guild_name, character_name, item_id) +); + +-- REVIEW (applied): the SQLite index was (guild_name) alone but every +-- query is world-scoped — composite matches the access path. +CREATE INDEX idx_watch_guild ON item_watch (world, guild_name); + +-- Persistent, admin-grantable roles (admin itself stays env-driven so a +-- DB wipe can't lock admins out; officer is computed from Census rank). +CREATE TABLE user_roles ( + discord_id text NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, + role text NOT NULL, + granted_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + granted_by text NOT NULL, -- discord_id of the granting admin + PRIMARY KEY (discord_id, role) +); + +CREATE INDEX idx_user_roles_role ON user_roles (role); + +-- Per-role capability map ("does this user have capability X?" joins +-- user_roles on role). New capability = INSERT rows, not schema. +CREATE TABLE role_permissions ( + role text NOT NULL, + capability text NOT NULL, + PRIMARY KEY (role, capability) +); + +CREATE INDEX idx_role_permissions_capability ON role_permissions (capability); + +-- Self-service role requests (claims-queue pattern: pending / approved / +-- rejected / withdrawn; approval also writes user_roles so the immutable +-- request history and the revocable grant stay decoupled). +CREATE TABLE role_requests ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + discord_id text NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, + role text NOT NULL, + status text NOT NULL DEFAULT 'pending', + requested_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + reviewed_at bigint, + reviewed_by text, -- admin's discord_id + user_note text, -- "why I want this" note + admin_note text -- admin's response note +); + +CREATE INDEX idx_role_requests_status ON role_requests (status); +CREATE INDEX idx_role_requests_discord ON role_requests (discord_id); +-- Only one pending request per (user, role); resolved ones coexist as audit. +CREATE UNIQUE INDEX idx_role_requests_one_pending + ON role_requests (discord_id, role) WHERE status = 'pending'; + +CREATE TABLE api_tokens ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + user_id text NOT NULL REFERENCES users(discord_id) DEFERRABLE INITIALLY IMMEDIATE, + name text NOT NULL, -- user-given label e.g. "Desktop ACT" + token_hash text NOT NULL UNIQUE, -- sha256 hex of the raw token + token_prefix text NOT NULL, -- first 12 chars for UI display + created_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + last_used_at bigint, -- updated on each successful auth + revoked_at bigint -- non-NULL = inactive +); + +CREATE INDEX idx_tokens_user ON api_tokens (user_id); +-- REVIEW (applied): SQLite also had idx_tokens_hash — redundant with the +-- UNIQUE constraint's index. Dropped. + +CREATE TABLE servers ( + world text PRIMARY KEY, + subdomain text NOT NULL UNIQUE, + display_name text NOT NULL, + max_level integer NOT NULL, + current_xpac text, + launch_dt text, + next_xpac text, + next_xpac_dt text, + -- Stamped by the automatic xpac rollover; the rankings era-lock cutoff. + current_xpac_started_dt text, + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + is_default integer NOT NULL DEFAULT 0 +); + +-- Guild raid schedules: up to 4 teams per guild, 4 raids per team. +CREATE TABLE raid_teams ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + world text NOT NULL, + guild_name text NOT NULL, + team_index integer NOT NULL, -- 0..3 (display order) + name text NOT NULL, + primary_tz text NOT NULL, -- IANA tz, e.g. America/New_York + twitch_login text, -- normalized channel login + updated_by text NOT NULL REFERENCES users(discord_id) DEFERRABLE INITIALLY IMMEDIATE, + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + UNIQUE (world, guild_name, team_index) +); + +CREATE INDEX idx_raid_teams_guild ON raid_teams (world, guild_name); + +CREATE TABLE raid_slots ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + team_id bigint NOT NULL REFERENCES raid_teams(id) ON DELETE CASCADE, + slot_index integer NOT NULL, -- 0..3 + days text NOT NULL, -- CSV of ISO weekdays (1=Mon..7=Sun); int[] conversion escalated to review + start_min integer NOT NULL, -- minutes since midnight in the team tz + end_min integer NOT NULL, -- may cross midnight; span <= 300 (5h) + label text, + UNIQUE (team_id, slot_index) +); + +CREATE INDEX idx_raid_slots_team ON raid_slots (team_id); + +-- Favourite (bookmark) characters — not ownership. Names are validated + +-- capitalised at the route layer so the plain UNIQUE is case-insensitive +-- in practice. +CREATE TABLE character_favorites ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + discord_id text NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, + character_name text NOT NULL, -- canonical capitalised EQ2 name + world text NOT NULL, + created_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + UNIQUE (discord_id, character_name, world) +); + +CREATE INDEX idx_favorites_character ON character_favorites (character_name, world); + +-- Download counters: one row per (user, slug) so public counts are +-- distinct-downloaders. +CREATE TABLE download_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + discord_id text NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, + slug text NOT NULL, + created_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + UNIQUE (discord_id, slug) +); + +CREATE INDEX idx_download_events_slug ON download_events (slug); + +-- Raid roster: which guild characters raid and in what capacity. +CREATE TABLE raid_roster_roles ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + world text NOT NULL, + guild_name text NOT NULL, + character_name text NOT NULL, + role text NOT NULL, -- raider or raid_alt + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + updated_by text, -- officer discord id (tombstoned by erasure) + placeholder integer NOT NULL DEFAULT 0, -- 1 = census-hidden character added by hand + cls text, -- class label for placeholder rows + UNIQUE (world, guild_name, character_name) +); + +CREATE INDEX idx_raid_roles_guild ON raid_roster_roles (world, guild_name); + +-- Canonical merged attendance per guild raid night. +-- REVIEW (applied): zones + uploaders become jsonb — erasure's LIKE probe +-- becomes an exact jsonb key-exists test, and the snapshot merge can later +-- become an atomic `uploaders || excluded.uploaders`. +CREATE TABLE attendance_sessions ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + world text NOT NULL, + guild_name text NOT NULL, -- canonical Census casing + session_day text NOT NULL, -- ISO date bucket (6h evening rollover) + seq integer NOT NULL DEFAULT 0, + started_at bigint NOT NULL, + ended_at bigint NOT NULL, + zones jsonb NOT NULL DEFAULT '[]', + scheduled integer NOT NULL DEFAULT 0, + team_index integer, + uploaders jsonb NOT NULL DEFAULT '{}', -- audit of contributing discord ids + created_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + UNIQUE (world, guild_name, session_day, seq) +); + +CREATE INDEX idx_att_sessions_guild ON attendance_sessions (world, guild_name, started_at); + +CREATE TABLE attendance_observations ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + session_id bigint NOT NULL REFERENCES attendance_sessions(id) ON DELETE CASCADE, + character_name text NOT NULL, -- canonical capitalised EQ2 name (voice rows carry a discord id) + kind text NOT NULL, -- raid / online / voice + first_seen bigint NOT NULL, + last_seen bigint NOT NULL, + UNIQUE (session_id, character_name, kind) +); + +-- Team layouts keyed by team_index (team ids regenerate on schedule saves — +-- the unkeyed reference is deliberate). +CREATE TABLE raid_placements ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + world text NOT NULL, + guild_name text NOT NULL, + team_index integer NOT NULL, -- 0..3 + character_name text NOT NULL, + group_num integer, -- 1..4; NULL when benched/sat out + slot integer, -- 0..5 within the group + sitout integer NOT NULL DEFAULT 0, + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + updated_by text, + UNIQUE (world, guild_name, team_index, character_name) +); + +CREATE INDEX idx_raid_place_team ON raid_placements (world, guild_name, team_index); + +-- Per-user availability calendar; absent row = Available. +CREATE TABLE user_availability ( + discord_id text NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, + day text NOT NULL, -- ISO date YYYY-MM-DD + status text NOT NULL, -- tentative or afk + updated_at bigint NOT NULL DEFAULT 0, -- newest-wins vs officer char entries + PRIMARY KEY (discord_id, day) +); + +-- Officer-set per-CHARACTER availability (lower-cased names, world-scoped). +CREATE TABLE character_availability ( + world text NOT NULL, + character_name text NOT NULL, -- lower-cased + day text NOT NULL, + status text NOT NULL, -- available / tentative / afk + set_by text NOT NULL, -- officer discord id + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + PRIMARY KEY (world, character_name, day) +); + +-- Saved AA planner builds. allocations stays text (opaque JSON round-trip; +-- jsonb buys nothing here). +CREATE TABLE aa_plans ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + discord_id text NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, + world text NOT NULL, + character_name text NOT NULL, + name text NOT NULL, + xpac text, + allocations text NOT NULL DEFAULT '{}', + share_slug text NOT NULL UNIQUE, + created_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) +); + +CREATE INDEX idx_aa_plans_owner ON aa_plans (discord_id, world, character_name); + +-- Discord server <-> EQ2 guild registry (bot /lexicon command group). +CREATE TABLE discord_guild_links ( + discord_guild_id text PRIMARY KEY, -- Discord snowflake as text + world text NOT NULL, + guild_name text NOT NULL, + voice_channel_id text, -- NULL = voice polling off + linked_by text NOT NULL, -- discord id of the linker (tombstoned by erasure) + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + parses_channel_id text, -- NULL = parse posting off + parses_posted_at bigint NOT NULL DEFAULT 0 +); + +-- Officer corrections to derived attendance categories. +CREATE TABLE attendance_overrides ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + session_id bigint NOT NULL REFERENCES attendance_sessions(id) ON DELETE CASCADE, + character_name text NOT NULL, + category text NOT NULL, -- present/sat_out/afk/awol/absent + set_by text NOT NULL, + set_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + UNIQUE (session_id, character_name) +); + +-- Officer-authored attendance timelines (timed periods). +CREATE TABLE attendance_segments ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + session_id bigint NOT NULL REFERENCES attendance_sessions(id) ON DELETE CASCADE, + character_name text NOT NULL, + category text NOT NULL, -- present/sat_out/afk + started_at bigint NOT NULL, + ended_at bigint NOT NULL, + set_by text NOT NULL, + set_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) +); + +CREATE INDEX idx_attendance_segments_session ON attendance_segments (session_id); + +-- Per-guild feature switches (leader-or-admin editable); absent row = defaults. +CREATE TABLE guild_settings ( + world text NOT NULL, + guild_name text NOT NULL, + officers_can_delete_parses integer NOT NULL DEFAULT 1, + updated_by text, + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + PRIMARY KEY (world, guild_name) +); + +-- Site-wide key/value settings (admin page). +CREATE TABLE site_settings ( + key text PRIMARY KEY, + value text NOT NULL, + updated_by text NOT NULL, + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) +); + +-- Guild recruitment profiles, keyed by census guild id (rename-proof). +-- The logo rides as bytea so it stays transactional with its attribution +-- columns (and inside the same backups as everything else). +CREATE TABLE guild_recruitment ( + world text NOT NULL, + guild_id bigint NOT NULL, -- census guild id (rename-proof anchor) + guild_name text NOT NULL, -- current name, verbatim casing (kept fresh) + recruiting integer NOT NULL DEFAULT 0, + description text NOT NULL DEFAULT '', + classes_json text NOT NULL DEFAULT '[]', + tags_json text NOT NULL DEFAULT '[]', + contacts_json text NOT NULL DEFAULT '[]', + discord_url text, + updated_by text, + updated_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + logo bytea, + logo_media_type text, + logo_uploaded_by text, + logo_uploaded_at bigint, + PRIMARY KEY (world, guild_id) +); + +CREATE UNIQUE INDEX idx_guild_recruitment_name ON guild_recruitment (world, guild_name); +CREATE INDEX idx_guild_recruitment_listing ON guild_recruitment (world, recruiting); + +-- seeds +-- Idempotent — the test scratch-schema reset re-runs everything below this +-- marker after TRUNCATE. Production values are overwritten by the data copy. +INSERT INTO role_permissions (role, capability) VALUES ('contributor', 'edit_content') +ON CONFLICT DO NOTHING; +INSERT INTO servers (world, subdomain, display_name, max_level, is_default) +VALUES ('Varsoon', 'varsoon', 'Varsoon', 70, 1), ('Wuoshi', 'wuoshi', 'Wuoshi', 70, 0) +ON CONFLICT DO NOTHING; +-- Exactly-one-default fixup (mirrors the retired migrations.py logic). +UPDATE servers SET is_default = 1 +WHERE world = (SELECT MIN(world) FROM servers) + AND NOT EXISTS (SELECT 1 FROM servers WHERE is_default = 1); From f0740116bf221eba6eacfb1407cec92583ba0fd1 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 13:43:08 +0100 Subject: [PATCH 04/32] =?UTF-8?q?feat(pg):=20users=20schema=20review=20ite?= =?UTF-8?q?ms=20=E2=80=94=20one-approved-claim=20index,=20days=20int[]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-approved review changes: partial unique index enforcing ONE approved claim per (world, lower(character_name)) at the database level (verified: case-variant duplicate blocked with UniqueViolation; local dev data audits clean — prod audit runs at rehearsal via the copy script), and raid_slots.days becomes a real integer[] (API already speaks lists; the store/poller code change lands with the P1 raid_schedule swap; the copy script splits prod CSV). Co-Authored-By: Claude Fable 5 --- db/migrations/0001_users.sql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/db/migrations/0001_users.sql b/db/migrations/0001_users.sql index a72ef34c..2351cfbb 100644 --- a/db/migrations/0001_users.sql +++ b/db/migrations/0001_users.sql @@ -50,6 +50,12 @@ CREATE INDEX idx_claims_world ON character_claims (world); -- Ported from migrations.sql (post-ALTER home no longer needed — the -- two-phase schema/migrations split is dead on Postgres). CREATE INDEX idx_claims_primary ON character_claims (world, is_primary) WHERE is_primary = 1; +-- REVIEW (applied, user-approved): one APPROVED claim per character per +-- world, enforced by the database instead of only by the supersede flow. +-- The copy script audits prod data for violations before load (a +-- historical double-approval fails loudly at rehearsal, not silently). +CREATE UNIQUE INDEX idx_claims_one_approved + ON character_claims (world, lower(character_name)) WHERE status = 'approved'; CREATE TABLE item_watch ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, @@ -164,7 +170,11 @@ CREATE TABLE raid_slots ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, team_id bigint NOT NULL REFERENCES raid_teams(id) ON DELETE CASCADE, slot_index integer NOT NULL, -- 0..3 - days text NOT NULL, -- CSV of ISO weekdays (1=Mon..7=Sun); int[] conversion escalated to review + -- REVIEW (applied, user-approved): ISO weekdays (1=Mon..7=Sun) as a real + -- array instead of the SQLite CSV string — the API already speaks + -- lists, so only the store/poller layer changes (lands with the P1 + -- raid_schedule store swap; the copy script splits prod CSV). + days integer[] NOT NULL, start_min integer NOT NULL, -- minutes since midnight in the team tz end_min integer NOT NULL, -- may cross midnight; span <= 300 (5h) label text, From 8c83c3da9ae0b742ccc899b59ac7780801d6fd89 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 14:15:14 +0100 Subject: [PATCH 05/32] feat(pg): convert all 16 users-family stores to psycopg against the users schema PgStoreBase/SchemaBound seams in db_catalogue; every store's _db() now checks out a pooled psycopg connection scoped by SET search_path. SQL sidecars: %s params, strftime -> extract(epoch), COLLATE NOCASE -> lower()=lower(%s), {placeholders} IN-lists -> = ANY(%s), RETURNING ids read as dict rows. attendance swaps BEGIN IMMEDIATE for pg_advisory_xact_lock + jsonb zones/uploaders (Json writes, parsed reads); recruitment logo round-trips bytea via bytes(); raid_slots.days rides integer[] through api/raid_schedule + raid_live; raid_planning keeps NULLS FIRST ordering parity. Route layer: role_requests 409 now catches psycopg UniqueViolation, bot guild_context degrades on psycopg.Error. Verified by live smoke of all stores on local PG17 (fixed two conversion bugs it caught: a dropped select_claim_user_and_world block and %s-in-comment placeholder miscounts in claims.sql). Co-Authored-By: Claude Fable 5 --- backend/bot/guild_context.py | 7 +- backend/db_catalogue.py | 51 +++++++++++++- backend/pg.py | 17 ++++- backend/server/api/raid_schedule.py | 4 +- backend/server/api/role_requests.py | 5 +- backend/server/db/__init__.py | 10 ++- backend/server/db/aa_plans.py | 29 ++++---- backend/server/db/aa_plans.sql | 20 +++--- backend/server/db/attendance.py | 81 +++++++++++---------- backend/server/db/attendance.sql | 93 +++++++++++++------------ backend/server/db/availability.py | 28 ++++---- backend/server/db/availability.sql | 22 +++--- backend/server/db/claims.py | 55 +++++++-------- backend/server/db/claims.sql | 55 ++++++++------- backend/server/db/discord_links.py | 29 ++++---- backend/server/db/discord_links.sql | 24 ++++--- backend/server/db/downloads.py | 27 ++++--- backend/server/db/downloads.sql | 6 +- backend/server/db/favorites.py | 31 ++++----- backend/server/db/favorites.sql | 26 +++---- backend/server/db/guild_recruitment.py | 49 ++++++------- backend/server/db/guild_recruitment.sql | 28 ++++---- backend/server/db/guild_settings.py | 24 +++---- backend/server/db/guild_settings.sql | 13 ++-- backend/server/db/item_watch.py | 38 +++++----- backend/server/db/item_watch.sql | 20 +++--- backend/server/db/raid_planning.py | 35 +++++----- backend/server/db/raid_planning.sql | 28 ++++---- backend/server/db/raid_schedule.py | 39 +++++------ backend/server/db/raid_schedule.sql | 14 ++-- backend/server/db/servers.py | 55 ++++++++------- backend/server/db/servers.sql | 20 +++--- backend/server/db/site_settings.py | 26 ++++--- backend/server/db/site_settings.sql | 10 +-- backend/server/db/tokens.py | 57 ++++++++------- backend/server/db/tokens.sql | 16 ++--- backend/server/db/users.py | 81 ++++++++++----------- backend/server/db/users.sql | 51 +++++++------- backend/server/raid_live.py | 2 +- 39 files changed, 648 insertions(+), 578 deletions(-) diff --git a/backend/bot/guild_context.py b/backend/bot/guild_context.py index 989c648d..e0b0c2e6 100644 --- a/backend/bot/guild_context.py +++ b/backend/bot/guild_context.py @@ -10,9 +10,10 @@ from __future__ import annotations import logging -import sqlite3 from dataclasses import dataclass +import psycopg + from backend.server.db.discord_links import store as links_store _log = logging.getLogger(__name__) @@ -69,7 +70,9 @@ async def resolve_guild_context(discord_guild_id: int | None) -> GuildContext: return _FALLBACK try: link = await links_store.get_link(str(discord_guild_id)) - except sqlite3.OperationalError as exc: + # psycopg.Error covers both "schema not migrated yet" (UndefinedTable) + # and connection trouble — the bot racing the web lifespan either way. + except psycopg.Error as exc: _log.warning("[bot] guild-link lookup failed (%s) — using fallback world", exc) return _FALLBACK if link is None: diff --git a/backend/db_catalogue.py b/backend/db_catalogue.py index 270236fc..a8f3ac14 100644 --- a/backend/db_catalogue.py +++ b/backend/db_catalogue.py @@ -46,7 +46,7 @@ from collections.abc import AsyncIterator, Mapping, Sequence from contextlib import asynccontextmanager from pathlib import Path -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar if TYPE_CHECKING: import aiosqlite @@ -312,3 +312,52 @@ async def _db(self, *, row_factory: bool = False) -> AsyncIterator[aiosqlite.Con if row_factory: db.row_factory = aiosqlite.Row yield db + +# --------------------------------------------------------------------------- +# Postgres bases — the migrated families (users / parses / census / zones / +# raids). One database, one schema per family; connections come from +# backend.pg (pooled under the app lifespan, direct otherwise) and select +# their family schema via search_path at checkout, so every query in the +# .sql sidecars stays unqualified. Schema DDL is owned by db/migrations/ +# via backend.pg_migrate — stores never create tables. +# --------------------------------------------------------------------------- + + +class SchemaBound: + """Schema-name identity for Postgres stores — the migrated analog of + :class:`PathBound`. Tests re-point ``store.schema`` at leased scratch + schemas exactly as they re-pointed ``store.path`` at tmp files. No + ``__fspath__`` (nothing to open as a file) and no exists-degrade: + migrations run before the app serves, so a missing relation is a real + fault, never a soft empty result.""" + + def __init__(self, schema: str) -> None: + self.schema = schema + + def __repr__(self) -> str: # pragma: no cover — debugging nicety + return f"<{type(self).__name__} schema={self.schema}>" + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) and type(other) is type(self) and other.schema == self.schema + + def __hash__(self) -> int: + return hash((type(self), self.schema)) + + +class PgStoreBase(SchemaBound): + """Base for the async (formerly aiosqlite) users-family domain stores. + + ``_db()`` keeps its name and contextmanager shape so store methods port + with minimal churn; rows are ALWAYS dict-shaped (psycopg dict_row — the + ``row_factory`` kwarg survives for signature compatibility and is + ignored). psycopg commits on clean ``async with`` exit and rolls back + on exception, so the explicit ``await db.commit()`` calls inside store + methods remain correct (and make the commit point explicit).""" + + @asynccontextmanager + async def _db(self, *, row_factory: bool = True) -> AsyncIterator[Any]: + from backend import pg # deferred: sync-only consumers never pay the import + + async with pg.aconnection() as conn: + await conn.execute(pg.search_path_sql(self.schema)) + yield conn diff --git a/backend/pg.py b/backend/pg.py index 2b29e00d..97715e39 100644 --- a/backend/pg.py +++ b/backend/pg.py @@ -33,6 +33,17 @@ _DSN_VARS = ("DATABASE_URL", "SUPABASE_DB_URL", "POSTGRES_CONNECTION_STRING") +def ensure_selector_event_loop_policy() -> None: + """psycopg's async side cannot run on Windows' default ProactorEventLoop. + Call before asyncio.run() on entry points that touch Postgres (main.py, + conftest, scripts). No-op elsewhere; production Linux is unaffected.""" + import asyncio + import sys + + if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + def dsn() -> str: for var in _DSN_VARS: value = os.getenv(var) @@ -68,7 +79,9 @@ async def close_pools() -> None: @asynccontextmanager -async def aconnection() -> AsyncIterator[psycopg.AsyncConnection[Any]]: +async def aconnection() -> AsyncIterator[Any]: + # Yields Any on purpose: psycopg's stubs demand LiteralString queries, + # but the house pattern is named SQL blocks loaded from .sql sidecars. """Async checkout: pooled when the lifespan opened pools, else a short-lived direct connection (tests / scripts / pre-lifespan bot). psycopg commits on clean ``async with`` exit and rolls back on @@ -82,7 +95,7 @@ async def aconnection() -> AsyncIterator[psycopg.AsyncConnection[Any]]: @contextmanager -def connection() -> Iterator[psycopg.Connection[Any]]: +def connection() -> Iterator[Any]: """Sync twin of :func:`aconnection`.""" if _sync_pool is not None: with _sync_pool.connection() as conn: diff --git a/backend/server/api/raid_schedule.py b/backend/server/api/raid_schedule.py index 213e0f87..e3def17c 100644 --- a/backend/server/api/raid_schedule.py +++ b/backend/server/api/raid_schedule.py @@ -118,7 +118,7 @@ def _slot_to_db(slot: RaidSlotInput) -> dict: if span > _MAX_SPAN_MIN: raise HTTPException(status_code=400, detail="A raid can be at most 5 hours long.") label = sanitize_text(slot.label, max_len=_MAX_TEXT_LEN) or None - return {"days": ",".join(str(d) for d in days), "start_min": start, "end_min": end, "label": label} + return {"days": days, "start_min": start, "end_min": end, "label": label} def _screen_text(value: str, *, actor: str, guild: str, field: str) -> None: @@ -138,7 +138,7 @@ def _fmt_schedule(teams: list[dict]) -> RaidScheduleResponse: login = t.get("twitch_login") raids = [ RaidSlotResponse( - days=[int(d) for d in str(r["days"]).split(",") if d], + days=[int(d) for d in (r["days"] or [])], start_min=r["start_min"], end_min=r["end_min"], label=r.get("label"), diff --git a/backend/server/api/role_requests.py b/backend/server/api/role_requests.py index 43585a59..5e0140aa 100644 --- a/backend/server/api/role_requests.py +++ b/backend/server/api/role_requests.py @@ -12,8 +12,7 @@ from __future__ import annotations -import sqlite3 - +import psycopg.errors from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel, Field @@ -94,7 +93,7 @@ async def submit_role_request(body: SubmitRoleRequest, request: Request) -> Role raise HTTPException(status_code=409, detail=f"You already have the {body.role!r} role") try: new_id = await create_role_request(user["id"], body.role, body.note) - except sqlite3.IntegrityError as exc: + except psycopg.errors.UniqueViolation as exc: # The partial unique index on (discord_id, role) WHERE status='pending' # catches this — surface a clean 409 rather than a 500. raise HTTPException( diff --git a/backend/server/db/__init__.py b/backend/server/db/__init__.py index 2d307671..b030ba44 100644 --- a/backend/server/db/__init__.py +++ b/backend/server/db/__init__.py @@ -23,10 +23,16 @@ from backend.db_helpers import resolve_db_path from backend.server.db._assertions import assert_schema_complete from backend.server.db.migrations import apply_migrations -from backend.server.db.schema import SCHEMA +from backend.server.db.schema import SCHEMA as _SQLITE_SCHEMA DB_PATH = resolve_db_path("DB_USERS_PATH", "users.db") +#: Postgres schema the users family lives in (see db/migrations/ +#: 0001_users.sql). Tests re-point each store's ``schema`` at leased +#: scratch schemas; DB_PATH above survives only until the remaining +#: SQLite consumers (erasure's parses half) finish migrating. +SCHEMA = "users" + def init_db(path: Path | None = None) -> None: """Create tables if they don't exist + apply migrations. @@ -46,7 +52,7 @@ def init_db(path: Path | None = None) -> None: path = Path(path) if path is not None else DB_PATH path.parent.mkdir(parents=True, exist_ok=True) with sqlite3.connect(path) as conn: - conn.executescript(SCHEMA) + conn.executescript(_SQLITE_SCHEMA) apply_migrations(conn) assert_schema_complete(conn) diff --git a/backend/server/db/aa_plans.py b/backend/server/db/aa_plans.py index 9a69b2c2..595d08b8 100644 --- a/backend/server/db/aa_plans.py +++ b/backend/server/db/aa_plans.py @@ -1,4 +1,4 @@ -"""users.db aa_plans helpers (async aiosqlite). +"""users aa_plans helpers (async psycopg). Saved AA planner builds — owned by a Discord user, pinned to the character they were planned from, shareable read-only via the always-minted @@ -9,44 +9,43 @@ from __future__ import annotations import secrets -from pathlib import Path -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class AAPlansStore(AsyncStoreBase): - """users.db `aa_plans` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db).""" +class AAPlansStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def list_plans(self, discord_id: str, world: str, character_name: str) -> list[dict]: """The user's plans for one character, newest-updated first (summary rows — allocations excluded to keep the list light).""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_plans_for_character"], (discord_id, world, character_name)) as cur: + async with await db.execute(_SQL["select_plans_for_character"], (discord_id, world, character_name)) as cur: return [dict(r) for r in await cur.fetchall()] async def count_plans(self, discord_id: str, world: str, character_name: str) -> int: async with self._db() as db: - async with db.execute(_SQL["count_plans_for_character"], (discord_id, world, character_name)) as cur: + async with await db.execute(_SQL["count_plans_for_character"], (discord_id, world, character_name)) as cur: row = await cur.fetchone() - return int(row[0]) if row else 0 + return int(row["n"]) if row else 0 async def get_plan(self, plan_id: int) -> dict | None: async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_plan"], (plan_id,)) as cur: + async with await db.execute(_SQL["select_plan"], (plan_id,)) as cur: row = await cur.fetchone() return dict(row) if row else None async def get_plan_by_slug(self, slug: str) -> dict | None: async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_plan_by_slug"], (slug,)) as cur: + async with await db.execute(_SQL["select_plan_by_slug"], (slug,)) as cur: row = await cur.fetchone() return dict(row) if row else None @@ -68,7 +67,7 @@ async def create_plan( ) row = await cur.fetchone() await db.commit() - plan_id = row[0] if row else 0 + plan_id = row["id"] if row else 0 plan = await self.get_plan(int(plan_id or 0)) if plan is None: # pragma: no cover — insert+select on one path raise RuntimeError("aa_plan insert did not persist") diff --git a/backend/server/db/aa_plans.sql b/backend/server/db/aa_plans.sql index c39402c1..005e27ae 100644 --- a/backend/server/db/aa_plans.sql +++ b/backend/server/db/aa_plans.sql @@ -1,34 +1,34 @@ --- users.db aa_plans DML (schema lives in schema.sql — the package init_db --- orchestrator owns it). +-- SQL for backend/server/db/aa_plans.py (psycopg, users schema). Schema DDL +-- lives in db/migrations/0001_users.sql. -- :name select_plans_for_character SELECT id, name, xpac, share_slug, created_at, updated_at FROM aa_plans - WHERE discord_id = ? AND world = ? AND character_name = ? + WHERE discord_id = %s AND world = %s AND character_name = %s ORDER BY updated_at DESC; -- :name count_plans_for_character -SELECT COUNT(*) FROM aa_plans WHERE discord_id = ? AND world = ? AND character_name = ?; +SELECT COUNT(*) AS n FROM aa_plans WHERE discord_id = %s AND world = %s AND character_name = %s; -- :name select_plan SELECT id, discord_id, world, character_name, name, xpac, allocations, share_slug, created_at, updated_at FROM aa_plans - WHERE id = ?; + WHERE id = %s; -- :name select_plan_by_slug SELECT id, discord_id, world, character_name, name, xpac, allocations, share_slug, created_at, updated_at FROM aa_plans - WHERE share_slug = ?; + WHERE share_slug = %s; -- :name insert_plan INSERT INTO aa_plans (discord_id, world, character_name, name, xpac, allocations, share_slug) -VALUES (?, ?, ?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING id; -- :name update_plan UPDATE aa_plans - SET name = ?, allocations = ?, xpac = ?, updated_at = strftime('%s','now') - WHERE id = ? AND discord_id = ?; + SET name = %s, allocations = %s, xpac = %s, updated_at = floor(extract(epoch from now())) + WHERE id = %s AND discord_id = %s; -- :name delete_plan -DELETE FROM aa_plans WHERE id = ? AND discord_id = ?; +DELETE FROM aa_plans WHERE id = %s AND discord_id = %s; diff --git a/backend/server/db/attendance.py b/backend/server/db/attendance.py index f98e8a3e..dae352ee 100644 --- a/backend/server/db/attendance.py +++ b/backend/server/db/attendance.py @@ -1,4 +1,4 @@ -"""users.db raid-attendance store (async aiosqlite). +"""users-schema raid-attendance store (psycopg). Canonical merged attendance per guild raid night. Multiple officers run the parser simultaneously and each POSTs cumulative snapshots; ``apply_snapshot`` @@ -12,20 +12,21 @@ double-headers (>3h apart). The ``scheduled`` flag + ``team_index`` are frozen at ingest so later schedule edits never rewrite history. -Mirrors the favorites/raid_schedule domain pattern: per-call connections via -``AsyncStoreBase._db()``; tests re-point ``store.path`` via ALL_STORES. +Mirrors the guild_settings/raid_schedule domain pattern: pooled per-call +connections via ``PgStoreBase._db()``; tests re-point ``store.schema`` via +ALL_STORES. """ from __future__ import annotations -import json import time from collections.abc import Sequence from datetime import UTC, datetime -from pathlib import Path -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from psycopg.types.json import Json + +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) @@ -49,12 +50,13 @@ def session_day_for(started_at: int) -> str: return datetime.fromtimestamp(started_at - ROLLOVER_S, tz=UTC).date().isoformat() -class AttendanceStore(AsyncStoreBase): - """users.db `attendance` domain. Schema/migrations are owned by the - package orchestrator (backend.server.db.init_db).""" +class AttendanceStore(PgStoreBase): + """users-schema `attendance` domain. Schema DDL is owned by + db/migrations/0001_users.sql; methods check out pooled connections + scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def apply_snapshot( self, @@ -84,8 +86,17 @@ async def apply_snapshot( extend = len(raid_members) >= MIN_RAID_FOR_WINDOW async with self._db(row_factory=True) as db: - await db.execute("BEGIN IMMEDIATE") try: + # psycopg's implicit transaction opens at this first execute. + # The advisory xact lock (auto-released at commit/rollback) + # serialises concurrent snapshot folds per (world, guild) + # exactly as SQLite's BEGIN IMMEDIATE eager write lock did — + # two uploaders can't both miss the other's freshly inserted + # session and double-create. + await db.execute( + "SELECT pg_advisory_xact_lock(hashtext(%s))", + (f"attendance:{world.lower()}:{guild_name.lower()}",), + ) # Merge matching uses the full observation window so overnight # online rows still land in the right session… row = await ( @@ -108,16 +119,18 @@ async def apply_snapshot( if merged and row is not None: session_id = row["id"] session_day = row["session_day"] - zone_set = set(json.loads(row["zones"] or "[]")) | set(zones) - uploaders = json.loads(row["uploaders"] or "{}") + # zones/uploaders are jsonb — psycopg hands back the + # parsed list/dict directly, no json.loads round-trip. + zone_set = set(row["zones"] or []) | set(zones) + uploaders = row["uploaders"] or {} uploaders[discord_id] = sent_at await db.execute( _SQL["merge_session_window"], ( win_start if extend else row["started_at"], win_end if extend else row["ended_at"], - json.dumps(sorted(zone_set)), - json.dumps(uploaders), + Json(sorted(zone_set)), + Json(uploaders), 1 if scheduled else 0, team_index, session_id, @@ -128,7 +141,7 @@ async def apply_snapshot( seq_row = await ( await db.execute(_SQL["select_max_seq"], (world, guild_name, session_day)) ).fetchone() - seq = seq_row[0] if seq_row else 0 # aggregate always returns a row + seq = seq_row["seq"] if seq_row else 0 # aggregate always returns a row cur = await db.execute( _SQL["insert_session"], ( @@ -138,14 +151,14 @@ async def apply_snapshot( seq, win_start, win_end, - json.dumps(sorted(set(zones))), + Json(sorted(set(zones))), 1 if scheduled else 0, team_index, - json.dumps({discord_id: sent_at}), + Json({discord_id: sent_at}), ), ) _row = await cur.fetchone() - session_id = _row[0] if _row else None + session_id = _row["id"] if _row else None for kind, members in (("raid", raid_members), ("online", online_guildies)): for m in members: @@ -164,7 +177,7 @@ async def find_live_session(self, world: str, guild_name: str, at: int) -> dict voice poller's "is a raid happening right now?" probe. None when nothing is within the merge gap.""" async with self._db(row_factory=True) as db: - async with db.execute( + async with await db.execute( _SQL["select_live_session"], (world, guild_name, at + MERGE_GAP_S, at - MERGE_GAP_S), ) as cur: @@ -187,28 +200,28 @@ async def list_sessions( self, world: str, guild_name: str, *, limit: int = 50, before_id: int | None = None ) -> list[dict]: async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_sessions"], (world, guild_name, before_id, before_id, limit)) as cur: + async with await db.execute( + _SQL["select_sessions"], (world, guild_name, before_id, before_id, limit) + ) as cur: return [dict(r) for r in await cur.fetchall()] async def get_session(self, session_id: int) -> dict | None: async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_session"], (session_id,)) as cur: + async with await db.execute(_SQL["select_session"], (session_id,)) as cur: row = await cur.fetchone() return dict(row) if row else None async def observations_for_session(self, session_id: int) -> list[dict]: async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_observations"], (session_id,)) as cur: + async with await db.execute(_SQL["select_observations"], (session_id,)) as cur: return [dict(r) for r in await cur.fetchall()] async def observations_for_sessions(self, session_ids: list[int]) -> dict[int, list[dict]]: if not session_ids: return {} - placeholders = ",".join("?" * len(session_ids)) out: dict[int, list[dict]] = {sid: [] for sid in session_ids} async with self._db(row_factory=True) as db: - sql = _SQL["select_observations_many"].format(placeholders=placeholders) - async with db.execute(sql, session_ids) as cur: + async with await db.execute(_SQL["select_observations_many"], (session_ids,)) as cur: for r in await cur.fetchall(): out[r["session_id"]].append(dict(r)) return out @@ -232,17 +245,15 @@ async def clear_override(self, session_id: int, character_name: str) -> bool: async def overrides_for_session(self, session_id: int) -> dict[str, dict]: """{character_name_lower: {character_name, category, set_by, set_at}}.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_overrides"], (session_id,)) as cur: + async with await db.execute(_SQL["select_overrides"], (session_id,)) as cur: return {r["character_name"].lower(): dict(r) for r in await cur.fetchall()} async def overrides_for_sessions(self, session_ids: list[int]) -> dict[int, dict[str, dict]]: if not session_ids: return {} - placeholders = ",".join("?" * len(session_ids)) out: dict[int, dict[str, dict]] = {sid: {} for sid in session_ids} async with self._db(row_factory=True) as db: - sql = _SQL["select_overrides_many"].format(placeholders=placeholders) - async with db.execute(sql, session_ids) as cur: + async with await db.execute(_SQL["select_overrides_many"], (session_ids,)) as cur: for r in await cur.fetchall(): out[r["session_id"]][r["character_name"].lower()] = dict(r) return out @@ -275,7 +286,7 @@ async def segments_for_session(self, session_id: int) -> dict[str, list[dict]]: ended_at, set_by}]} ordered by started_at.""" out: dict[str, list[dict]] = {} async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_segments"], (session_id,)) as cur: + async with await db.execute(_SQL["select_segments"], (session_id,)) as cur: for r in await cur.fetchall(): out.setdefault(r["character_name"].lower(), []).append(dict(r)) return out @@ -283,11 +294,9 @@ async def segments_for_session(self, session_id: int) -> dict[str, list[dict]]: async def segments_for_sessions(self, session_ids: list[int]) -> dict[int, dict[str, list[dict]]]: if not session_ids: return {} - placeholders = ",".join("?" * len(session_ids)) out: dict[int, dict[str, list[dict]]] = {sid: {} for sid in session_ids} async with self._db(row_factory=True) as db: - sql = _SQL["select_segments_many"].format(placeholders=placeholders) - async with db.execute(sql, session_ids) as cur: + async with await db.execute(_SQL["select_segments_many"], (session_ids,)) as cur: for r in await cur.fetchall(): out[r["session_id"]].setdefault(r["character_name"].lower(), []).append(dict(r)) return out diff --git a/backend/server/db/attendance.sql b/backend/server/db/attendance.sql index 3a4cfcaf..6b235681 100644 --- a/backend/server/db/attendance.sql +++ b/backend/server/db/attendance.sql @@ -1,4 +1,4 @@ --- SQL for backend/server/db/attendance.py (async aiosqlite). +-- SQL for backend/server/db/attendance.py (psycopg, users schema). -- Find the newest session whose window overlaps the incoming snapshot's -- window widened by the merge gap (params: world, guild, win_end + gap, @@ -6,133 +6,138 @@ -- :name select_overlapping_session SELECT id, session_day, seq, started_at, ended_at, zones, scheduled, team_index, uploaders FROM attendance_sessions -WHERE world = ? AND guild_name = ? AND started_at <= ? AND ended_at >= ? +WHERE world = %s AND guild_name = %s AND started_at <= %s AND ended_at >= %s ORDER BY started_at DESC LIMIT 1; -- Point-in-time probe for the voice poller: the session a snapshot at time -- T WOULD merge into (select_overlapping_session with a zero-width window, --- widened by the merge gap in Python). NOCASE both keys — the /lexicon --- registry stores canonical casing but belt-and-braces costs nothing. +-- widened by the merge gap in Python). Case-insensitive on both keys — the +-- /lexicon registry stores canonical casing but belt-and-braces costs +-- nothing. -- :name select_live_session SELECT id, session_day, started_at, ended_at FROM attendance_sessions -WHERE world = ? COLLATE NOCASE AND guild_name = ? COLLATE NOCASE - AND started_at <= ? AND ended_at >= ? +WHERE lower(world) = lower(%s) AND lower(guild_name) = lower(%s) + AND started_at <= %s AND ended_at >= %s ORDER BY started_at DESC LIMIT 1; -- :name select_max_seq -SELECT COALESCE(MAX(seq) + 1, 0) FROM attendance_sessions -WHERE world = ? AND guild_name = ? AND session_day = ?; +SELECT COALESCE(MAX(seq) + 1, 0) AS seq FROM attendance_sessions +WHERE world = %s AND guild_name = %s AND session_day = %s; -- :name insert_session INSERT INTO attendance_sessions (world, guild_name, session_day, seq, started_at, ended_at, zones, scheduled, team_index, uploaders) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id; -- :name merge_session_window UPDATE attendance_sessions - SET started_at = MIN(started_at, ?), ended_at = MAX(ended_at, ?), - zones = ?, uploaders = ?, scheduled = MAX(scheduled, ?), - team_index = COALESCE(team_index, ?), - updated_at = strftime('%s','now') - WHERE id = ?; + SET started_at = LEAST(started_at, %s), ended_at = GREATEST(ended_at, %s), + zones = %s, uploaders = %s, scheduled = GREATEST(scheduled, %s), + team_index = COALESCE(team_index, %s), + updated_at = floor(extract(epoch from now())) + WHERE id = %s; -- Commutative min/max upsert — uploader arrival order is irrelevant. -- :name upsert_observation INSERT INTO attendance_observations (session_id, character_name, kind, first_seen, last_seen) -VALUES (?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s) ON CONFLICT(session_id, character_name, kind) DO UPDATE SET - first_seen = MIN(first_seen, excluded.first_seen), - last_seen = MAX(last_seen, excluded.last_seen); + first_seen = LEAST(attendance_observations.first_seen, excluded.first_seen), + last_seen = GREATEST(attendance_observations.last_seen, excluded.last_seen); +-- The before_id probe parameter needs the ::bigint cast — it appears only +-- in an IS NULL test, so Postgres can't infer its type on its own. -- :name select_sessions SELECT id, session_day, seq, started_at, ended_at, zones, scheduled, team_index FROM attendance_sessions -WHERE world = ? AND guild_name = ? AND (? IS NULL OR id < ?) -ORDER BY id DESC LIMIT ?; +WHERE world = %s AND guild_name = %s AND (%s::bigint IS NULL OR id < %s) +ORDER BY id DESC LIMIT %s; -- :name select_session SELECT id, world, guild_name, session_day, seq, started_at, ended_at, zones, scheduled, team_index, uploaders -FROM attendance_sessions WHERE id = ?; +FROM attendance_sessions WHERE id = %s; -- :name select_observations SELECT session_id, character_name, kind, first_seen, last_seen -FROM attendance_observations WHERE session_id = ?; +FROM attendance_observations WHERE session_id = %s; -- :name select_observations_many --- {placeholders} = comma-joined "?" list composed in Python. +-- The id list binds as ONE array parameter (= ANY) — no composed +-- placeholder strings and no variable-count limits. SELECT session_id, character_name, kind, first_seen, last_seen -FROM attendance_observations WHERE session_id IN ({placeholders}); +FROM attendance_observations WHERE session_id = ANY(%s); -- Officer category corrections (see attendance_overrides schema comment). -- :name upsert_override INSERT INTO attendance_overrides (session_id, character_name, category, set_by, set_at) -VALUES (?, ?, ?, ?, strftime('%s','now')) +VALUES (%s, %s, %s, %s, floor(extract(epoch from now()))) ON CONFLICT(session_id, character_name) DO UPDATE SET category = excluded.category, set_by = excluded.set_by, set_at = excluded.set_at; -- :name delete_override -DELETE FROM attendance_overrides WHERE session_id = ? AND LOWER(character_name) = LOWER(?); +DELETE FROM attendance_overrides WHERE session_id = %s AND LOWER(character_name) = LOWER(%s); -- :name select_overrides SELECT character_name, category, set_by, set_at -FROM attendance_overrides WHERE session_id = ?; +FROM attendance_overrides WHERE session_id = %s; -- :name select_overrides_many --- {placeholders} = comma-joined "?" list composed in Python. +-- The id list binds as ONE array parameter (= ANY). SELECT session_id, character_name, category, set_by, set_at -FROM attendance_overrides WHERE session_id IN ({placeholders}); +FROM attendance_overrides WHERE session_id = ANY(%s); -- Officer window correction — start/end only; day/seq grouping is frozen. -- :name update_session_window UPDATE attendance_sessions - SET started_at = ?, ended_at = ?, updated_at = strftime('%s','now') - WHERE id = ?; + SET started_at = %s, ended_at = %s, updated_at = floor(extract(epoch from now())) + WHERE id = %s; -- Officer-authored timelines (attendance_segments): a character's manual -- rows replace their derived timeline. Replace = delete + insert in one tx. -- :name delete_segments_for_character -DELETE FROM attendance_segments WHERE session_id = ? AND LOWER(character_name) = LOWER(?); +DELETE FROM attendance_segments WHERE session_id = %s AND LOWER(character_name) = LOWER(%s); -- :name insert_segment INSERT INTO attendance_segments (session_id, character_name, category, started_at, ended_at, set_by) -VALUES (?, ?, ?, ?, ?, ?); +VALUES (%s, %s, %s, %s, %s, %s); -- :name select_segments SELECT character_name, category, started_at, ended_at, set_by -FROM attendance_segments WHERE session_id = ? ORDER BY started_at; +FROM attendance_segments WHERE session_id = %s ORDER BY started_at; -- :name select_segments_many --- {placeholders} = comma-joined "?" list composed in Python. +-- The id list binds as ONE array parameter (= ANY). SELECT session_id, character_name, category, started_at, ended_at, set_by -FROM attendance_segments WHERE session_id IN ({placeholders}) ORDER BY started_at; +FROM attendance_segments WHERE session_id = ANY(%s) ORDER BY started_at; -- :name delete_segments_for_session -DELETE FROM attendance_segments WHERE session_id = ?; +DELETE FROM attendance_segments WHERE session_id = %s; -- Row removal for junk names (mis-parses): kill the character's raid/online -- observations AND any override in one officer action. Voice rows key by -- discord id, not character name — untouched by design. -- :name delete_character_observations DELETE FROM attendance_observations -WHERE session_id = ? AND LOWER(character_name) = LOWER(?) AND kind IN ('raid', 'online'); +WHERE session_id = %s AND LOWER(character_name) = LOWER(%s) AND kind IN ('raid', 'online'); --- users.db connections don't enable PRAGMA foreign_keys (see --- AsyncStoreBase._db), so the schema's ON DELETE CASCADE never fires — --- observations are deleted explicitly in the same transaction. +-- The Postgres schema's ON DELETE CASCADE would cover these, but the +-- child rows are still deleted explicitly in the same transaction — +-- explicit beats implicit for an officer-facing destructive action, and +-- it keeps parity with the SQLite-era behaviour. -- :name delete_observations_for_session -DELETE FROM attendance_observations WHERE session_id = ?; +DELETE FROM attendance_observations WHERE session_id = %s; -- :name delete_overrides_for_session -DELETE FROM attendance_overrides WHERE session_id = ?; +DELETE FROM attendance_overrides WHERE session_id = %s; -- :name delete_session -DELETE FROM attendance_sessions WHERE id = ?; +DELETE FROM attendance_sessions WHERE id = %s; -- Retention: voice rows carry DISCORD IDS of everyone in the raid voice -- channel, site user or not. They only feed the in_voice flag, so they are -- swept after VOICE_OBSERVATION_RETENTION_DAYS (the privacy policy says so). -- :name delete_stale_voice_observations -DELETE FROM attendance_observations WHERE kind = 'voice' AND last_seen < ?; +DELETE FROM attendance_observations WHERE kind = 'voice' AND last_seen < %s; diff --git a/backend/server/db/availability.py b/backend/server/db/availability.py index a61d14a8..8c1ee0f2 100644 --- a/backend/server/db/availability.py +++ b/backend/server/db/availability.py @@ -1,4 +1,4 @@ -"""users.db per-user raid-availability calendar (async aiosqlite). +"""users per-user raid-availability calendar (async psycopg). Only non-default days are stored — an absent row means Available, so a player who never touches the calendar is always available (per the raid- @@ -11,10 +11,8 @@ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) @@ -50,17 +48,17 @@ def merge_availability( return {n: s for n, (s, _) in merged.items()} -class AvailabilityStore(AsyncStoreBase): - """users.db `user_availability` domain. Schema/migrations are owned by - the package orchestrator (backend.server.db.init_db).""" +class AvailabilityStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def get_range(self, discord_id: str, from_day: str, to_day: str) -> dict[str, str]: """{YYYY-MM-DD: 'tentative'|'afk'} for the window. Absent = available.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_range"], (discord_id, from_day, to_day)) as cur: + async with await db.execute(_SQL["select_range"], (discord_id, from_day, to_day)) as cur: return {r["day"]: r["status"] for r in await cur.fetchall()} async def set_days(self, discord_id: str, days: dict[str, str]) -> None: @@ -81,7 +79,7 @@ async def statuses_for_day(self, day: str) -> dict[str, str]: """{discord_id: status} for every user with a non-default entry on ``day`` — the raid planner overlays this onto the claims map.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_statuses_for_day"], (day,)) as cur: + async with await db.execute(_SQL["select_statuses_for_day"], (day,)) as cur: return {r["discord_id"]: r["status"] for r in await cur.fetchall()} async def statuses_for_day_with_times(self, day: str) -> dict[str, tuple[str, int]]: @@ -89,7 +87,7 @@ async def statuses_for_day_with_times(self, day: str) -> dict[str, tuple[str, in merge against officer character entries. Legacy rows (pre-stamp) carry updated_at 0, so any stamped officer edit beats them.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_statuses_for_day_with_times"], (day,)) as cur: + async with await db.execute(_SQL["select_statuses_for_day_with_times"], (day,)) as cur: return {r["discord_id"]: (r["status"], r["updated_at"]) for r in await cur.fetchall()} async def set_character_days(self, world: str, character_name: str, days: dict[str, str], *, set_by: str) -> None: @@ -112,14 +110,14 @@ async def char_statuses_for_day(self, world: str, day: str) -> dict[str, str]: (``available`` rows included — consumers checking for 'afk' are unaffected; the planner merge drops them after precedence).""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["char_statuses_for_day"], (world, day)) as cur: + async with await db.execute(_SQL["char_statuses_for_day"], (world, day)) as cur: return {r["character_name"]: r["status"] for r in await cur.fetchall()} async def char_statuses_for_day_with_times(self, world: str, day: str) -> dict[str, tuple[str, int]]: """{character_name_lower: (status, updated_at)} — the officer half of the newest-edit-wins planner merge.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["char_statuses_for_day_with_times"], (world, day)) as cur: + async with await db.execute(_SQL["char_statuses_for_day_with_times"], (world, day)) as cur: return {r["character_name"]: (r["status"], r["updated_at"]) for r in await cur.fetchall()} diff --git a/backend/server/db/availability.sql b/backend/server/db/availability.sql index f0c8de0c..abfee16e 100644 --- a/backend/server/db/availability.sql +++ b/backend/server/db/availability.sql @@ -1,43 +1,43 @@ --- SQL for backend/server/db/availability.py (AvailabilityStore). +-- SQL for backend/server/db/availability.py (AvailabilityStore; psycopg, users schema). -- Stored 'available' rows are an explicit newest-wins clear, not calendar -- content — the calendar renders them as the default. -- :name select_range SELECT day, status FROM user_availability -WHERE discord_id = ? AND day >= ? AND day <= ? AND status != 'available' +WHERE discord_id = %s AND day >= %s AND day <= %s AND status != 'available' ORDER BY day; -- :name upsert_day INSERT INTO user_availability (discord_id, day, status, updated_at) -VALUES (?, ?, ?, strftime('%s','now')) +VALUES (%s, %s, %s, floor(extract(epoch from now()))) ON CONFLICT(discord_id, day) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at; -- :name delete_day -DELETE FROM user_availability WHERE discord_id = ? AND day = ?; +DELETE FROM user_availability WHERE discord_id = %s AND day = %s; -- :name select_statuses_for_day SELECT discord_id, status FROM user_availability -WHERE day = ?; +WHERE day = %s; -- :name select_statuses_for_day_with_times SELECT discord_id, status, updated_at FROM user_availability -WHERE day = ?; +WHERE day = %s; -- :name char_upsert_day INSERT INTO character_availability (world, character_name, day, status, set_by) -VALUES (?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s) ON CONFLICT(world, character_name, day) -DO UPDATE SET status = excluded.status, set_by = excluded.set_by, updated_at = strftime('%s','now'); +DO UPDATE SET status = excluded.status, set_by = excluded.set_by, updated_at = floor(extract(epoch from now())); -- :name char_delete_day -DELETE FROM character_availability WHERE world = ? AND character_name = ? AND day = ?; +DELETE FROM character_availability WHERE world = %s AND character_name = %s AND day = %s; -- :name char_statuses_for_day SELECT character_name, status FROM character_availability -WHERE world = ? AND day = ?; +WHERE world = %s AND day = %s; -- :name char_statuses_for_day_with_times SELECT character_name, status, updated_at FROM character_availability -WHERE world = ? AND day = ?; +WHERE world = %s AND day = %s; diff --git a/backend/server/db/claims.py b/backend/server/db/claims.py index 3f77e74f..7765dbdd 100644 --- a/backend/server/db/claims.py +++ b/backend/server/db/claims.py @@ -1,6 +1,6 @@ -"""users.db character_claims table helpers. +"""users ``character_claims`` helpers (async psycopg). -Carved out of the original 1309-line web/db.py. Async (aiosqlite) helpers +Carved out of the original 1309-line web/db.py. Async (psycopg) helpers for the character claims domain. Per-call connections open via the shared ``AsyncStoreBase._db()``; tests re-point ``store.path``. @@ -14,22 +14,19 @@ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class ClaimsStore(AsyncStoreBase): - """users.db `claims` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class ClaimsStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def get_active_claims( self, @@ -44,7 +41,7 @@ async def get_active_claims( primaries are completely independent. """ async with self._db(row_factory=True) as db: - async with db.execute( + async with await db.execute( _SQL["list_active_claims"], (discord_id, world), ) as cur: @@ -70,7 +67,7 @@ async def submit_claim( async with self._db(row_factory=True) as db: # Reject if this character name is already claimed (approved or pending) # by anyone *on this world* (EQ2 names are unique only within a server) - async with db.execute( + async with await db.execute( _SQL["check_character_name_taken"], (character_name, world), ) as cur: @@ -90,9 +87,9 @@ async def submit_claim( (discord_id, character_name, world), ) _row = await cur.fetchone() - new_id = _row[0] if _row else None + new_id = _row["id"] if _row else None await db.commit() - async with db.execute(_SQL["find_by_id"], (new_id,)) as cur2: + async with await db.execute(_SQL["find_by_id"], (new_id,)) as cur2: row = await cur2.fetchone() assert row is not None, "INSERT succeeded but SELECT returned nothing" return dict(row) @@ -112,13 +109,13 @@ async def withdraw_claim( """ async with self._db() as db: # Check if this claim is primary before withdrawing (and capture world from row) - async with db.execute( + async with await db.execute( _SQL["select_primary_and_world"], (claim_id, discord_id), ) as cur: row = await cur.fetchone() - was_primary = row is not None and row[0] == 1 - claim_world = row[1] if row is not None else world + was_primary = row is not None and row["is_primary"] == 1 + claim_world = row["world"] if row is not None else world cur = await db.execute( _SQL["withdraw_claim"], @@ -150,7 +147,7 @@ async def set_primary( """ async with self._db() as db: # Verify the claim belongs to this user, is approved, and is on the right world - async with db.execute( + async with await db.execute( _SQL["find_approved_claim_for_user_on_world"], (claim_id, discord_id, world), ) as cur: @@ -175,7 +172,7 @@ async def get_claim_by_id( ) -> dict | None: """Return a single claim joined with its submitting user's info.""" async with self._db(row_factory=True) as db: - async with db.execute( + async with await db.execute( _SQL["find_claim_with_user"], (claim_id,), ) as cur: @@ -198,14 +195,14 @@ async def list_claims( where_parts: list[str] = [] params: list = [] if status: - where_parts.append("c.status = ?") + where_parts.append("c.status = %s") params.append(status) if world is not None: - where_parts.append("c.world = ?") + where_parts.append("c.world = %s") params.append(world) where_sql = ("WHERE " + " AND ".join(where_parts)) if where_parts else "" order = "ASC" if status == "pending" else "DESC" - async with db.execute( + async with await db.execute( _SQL["list_claims"].format(where_sql=where_sql, order=order), params, ) as cur: @@ -229,13 +226,13 @@ async def review_claim( Returns the updated claim (with user info) or None if not found. """ async with self._db() as db: - async with db.execute(_SQL["select_claim_user_and_world"], (claim_id,)) as cur: + async with await db.execute(_SQL["select_claim_user_and_world"], (claim_id,)) as cur: row = await cur.fetchone() if not row: return None - discord_id = row[0] - claim_world = row[1] - character_name = row[2] + discord_id = row["discord_id"] + claim_world = row["world"] + character_name = row["character_name"] await db.execute( _SQL["review_claim"], @@ -243,7 +240,7 @@ async def review_claim( ) # Auto-assign primary if this is the user's first approved character on this world if status == "approved": - async with db.execute( + async with await db.execute( _SQL["check_user_has_primary_on_world"], (discord_id, claim_world, claim_id), ) as cur: diff --git a/backend/server/db/claims.sql b/backend/server/db/claims.sql index 4e4bef35..8f98441d 100644 --- a/backend/server/db/claims.sql +++ b/backend/server/db/claims.sql @@ -1,61 +1,63 @@ --- SQL for backend/server/db/claims.py (async aiosqlite). +-- SQL for backend/server/db/claims.py (psycopg, users schema). -- Claims are scoped to (discord_id, world). -- :name list_active_claims SELECT * FROM character_claims -WHERE discord_id = ? AND world = ? AND status IN ('approved', 'pending') +WHERE discord_id = %s AND world = %s AND status IN ('approved', 'pending') ORDER BY requested_at ASC, id ASC; -- :name check_character_name_taken SELECT discord_id FROM character_claims -WHERE character_name = ? AND world = ? AND status IN ('approved', 'pending'); +WHERE character_name = %s AND world = %s AND status IN ('approved', 'pending'); -- :name withdraw_pending_claims_on_world UPDATE character_claims SET status = 'withdrawn' -WHERE discord_id = ? AND world = ? AND status = 'pending'; +WHERE discord_id = %s AND world = %s AND status = 'pending'; -- :name submit_claim INSERT INTO character_claims (discord_id, character_name, status, world) -VALUES (?, ?, 'pending', ?) +VALUES (%s, %s, 'pending', %s) RETURNING id; -- :name find_by_id -SELECT * FROM character_claims WHERE id = ?; +SELECT * FROM character_claims WHERE id = %s; -- :name select_primary_and_world -SELECT is_primary, world FROM character_claims WHERE id = ? AND discord_id = ?; +SELECT is_primary, world FROM character_claims WHERE id = %s AND discord_id = %s; -- :name withdraw_claim UPDATE character_claims SET status = 'withdrawn', is_primary = 0 -WHERE id = ? AND discord_id = ? AND status IN ('pending', 'approved'); +WHERE id = %s AND discord_id = %s AND status IN ('pending', 'approved'); -- Promote the oldest remaining approved character on the same world to primary. -- :name promote_oldest_to_primary UPDATE character_claims SET is_primary = 1 WHERE id = ( SELECT id FROM character_claims - WHERE discord_id = ? AND world = ? AND status = 'approved' AND id != ? + WHERE discord_id = %s AND world = %s AND status = 'approved' AND id != %s ORDER BY requested_at ASC LIMIT 1 ); -- :name find_approved_claim_for_user_on_world -SELECT id FROM character_claims WHERE id = ? AND discord_id = ? AND world = ? AND status = 'approved'; +SELECT id FROM character_claims WHERE id = %s AND discord_id = %s AND world = %s AND status = 'approved'; -- :name clear_primary_on_world -UPDATE character_claims SET is_primary = 0 WHERE discord_id = ? AND world = ? AND status = 'approved'; +UPDATE character_claims SET is_primary = 0 WHERE discord_id = %s AND world = %s AND status = 'approved'; -- :name set_primary_by_id -UPDATE character_claims SET is_primary = 1 WHERE id = ?; +UPDATE character_claims SET is_primary = 1 WHERE id = %s; -- :name find_claim_with_user SELECT c.*, u.discord_name, u.discord_username, u.avatar FROM character_claims c LEFT JOIN users u ON u.discord_id = c.discord_id -WHERE c.id = ?; +WHERE c.id = %s; +-- {where_sql} and {order} are composed by Python. (Comment kept OUTSIDE the +-- block: psycopg's placeholder scan doesn't strip SQL comments, so a %s +-- formatted into an in-block comment would be counted as a parameter.) -- :name list_claims --- {where_sql} and {order} are composed by Python. SELECT c.*, u.discord_name, u.discord_username, u.avatar FROM character_claims c LEFT JOIN users u ON u.discord_id = c.discord_id @@ -63,30 +65,31 @@ LEFT JOIN users u ON u.discord_id = c.discord_id ORDER BY c.requested_at {order}; -- :name select_claim_user_and_world -SELECT discord_id, world, character_name FROM character_claims WHERE id = ?; +SELECT discord_id, world, character_name FROM character_claims WHERE id = %s; -- A user cannot favourite their own character; when a claim is approved the -- new owner's existing favourite of that character (if any) is removed in the --- same transaction. COLLATE NOCASE because the favourites row is stored --- capitalised while a claim may carry different casing. +-- same transaction. LOWER() on both sides (was SQLite COLLATE NOCASE) because +-- the favourites row is stored capitalised while a claim may carry different +-- casing. -- :name remove_new_owners_favorite DELETE FROM character_favorites -WHERE discord_id = ? AND world = ? AND character_name = ? COLLATE NOCASE; +WHERE discord_id = %s AND world = %s AND LOWER(character_name) = LOWER(%s); -- :name review_claim UPDATE character_claims -SET status = ?, - reviewed_at = strftime('%s','now'), - reviewed_by = ?, - note = ? -WHERE id = ?; +SET status = %s, + reviewed_at = floor(extract(epoch from now())), + reviewed_by = %s, + note = %s +WHERE id = %s; -- :name check_user_has_primary_on_world SELECT id FROM character_claims -WHERE discord_id = ? AND world = ? AND status = 'approved' AND is_primary = 1 AND id != ?; +WHERE discord_id = %s AND world = %s AND status = 'approved' AND is_primary = 1 AND id != %s; -- :name delete_claim -DELETE FROM character_claims WHERE id = ?; +DELETE FROM character_claims WHERE id = %s; -- :name delete_claims_for_user -DELETE FROM character_claims WHERE discord_id = ?; +DELETE FROM character_claims WHERE discord_id = %s; diff --git a/backend/server/db/discord_links.py b/backend/server/db/discord_links.py index 84512d94..63844375 100644 --- a/backend/server/db/discord_links.py +++ b/backend/server/db/discord_links.py @@ -1,4 +1,4 @@ -"""users.db discord_guild_links helpers (async aiosqlite). +"""users-schema discord_guild_links helpers (psycopg). Maps a Discord server (guild) to an EQ2 (world, guild_name) pair — the bot's per-Discord-guild context. Configured in Discord via the /lexicon @@ -6,29 +6,28 @@ command (backend/bot/guild_context.py) and by the Phase 3 voice-attendance poller (voice_channel_id = the raid voice channel to snapshot). -Mirrors the downloads domain: per-call connections via -``AsyncStoreBase._db()``; tests re-point ``store.path`` (conftest does it +Mirrors the guild_settings domain: pooled per-call connections via +``PgStoreBase._db()``; tests re-point ``store.schema`` (conftest does it for every ``ALL_STORES`` entry). The bot imports this store directly — no facade aliases. """ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class DiscordLinksStore(AsyncStoreBase): - """users.db `discord_guild_links` domain. Schema is owned by the package - orchestrator (backend.server.db.init_db).""" +class DiscordLinksStore(PgStoreBase): + """users-schema `discord_guild_links` domain. Schema DDL is owned by + db/migrations/0001_users.sql; methods check out pooled connections + scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def upsert_link(self, discord_guild_id: str, world: str, guild_name: str, linked_by: str) -> None: """Create or update a Discord-guild → EQ2-guild mapping. Relinking @@ -48,7 +47,7 @@ async def set_voice_channel(self, discord_guild_id: str, voice_channel_id: str | async def get_link(self, discord_guild_id: str) -> dict | None: async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_link"], (discord_guild_id,)) as cur: + async with await db.execute(_SQL["select_link"], (discord_guild_id,)) as cur: row = await cur.fetchone() return dict(row) if row else None @@ -62,7 +61,7 @@ async def list_voice_links(self) -> list[dict]: """Every link with voice polling configured — the voice poller's per-tick work list.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_voice_links"]) as cur: + async with await db.execute(_SQL["select_voice_links"]) as cur: return [dict(r) for r in await cur.fetchall()] async def set_parses_channel(self, discord_guild_id: str, channel_id: str | None) -> bool: @@ -84,7 +83,7 @@ async def list_parse_links(self) -> list[dict]: """Every link with parse posting configured — the parse poster's per-tick work list.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_parse_links"]) as cur: + async with await db.execute(_SQL["select_parse_links"]) as cur: return [dict(r) for r in await cur.fetchall()] diff --git a/backend/server/db/discord_links.sql b/backend/server/db/discord_links.sql index e28d95a8..e6e88f0a 100644 --- a/backend/server/db/discord_links.sql +++ b/backend/server/db/discord_links.sql @@ -1,10 +1,10 @@ --- SQL for backend/server/db/discord_links.py (async aiosqlite). +-- SQL for backend/server/db/discord_links.py (psycopg, users schema). -- Relinking updates the mapping but deliberately PRESERVES voice_channel_id -- — an officer fixing a typo'd guild name shouldn't lose the voice config. -- :name upsert_link INSERT INTO discord_guild_links (discord_guild_id, world, guild_name, linked_by, updated_at) -VALUES (?, ?, ?, ?, strftime('%s','now')) +VALUES (%s, %s, %s, %s, floor(extract(epoch from now()))) ON CONFLICT(discord_guild_id) DO UPDATE SET world = excluded.world, guild_name = excluded.guild_name, @@ -13,16 +13,16 @@ ON CONFLICT(discord_guild_id) DO UPDATE SET -- :name set_voice_channel UPDATE discord_guild_links - SET voice_channel_id = ?, updated_at = strftime('%s','now') - WHERE discord_guild_id = ?; + SET voice_channel_id = %s, updated_at = floor(extract(epoch from now())) + WHERE discord_guild_id = %s; -- :name select_link SELECT discord_guild_id, world, guild_name, voice_channel_id, linked_by, updated_at, parses_channel_id, parses_posted_at -FROM discord_guild_links WHERE discord_guild_id = ?; +FROM discord_guild_links WHERE discord_guild_id = %s; -- :name delete_link -DELETE FROM discord_guild_links WHERE discord_guild_id = ?; +DELETE FROM discord_guild_links WHERE discord_guild_id = %s; -- Every link with voice polling on — the Phase 3 poller's work list. -- :name select_voice_links @@ -31,15 +31,17 @@ FROM discord_guild_links WHERE voice_channel_id IS NOT NULL; -- Enabling parse posting stamps the watermark to "now" so the poller never -- floods the channel with history; clearing leaves the watermark alone. +-- The CASE's probe parameter needs the ::text cast — it appears only in an +-- IS NULL test, so Postgres can't infer its type on its own. -- :name set_parses_channel UPDATE discord_guild_links - SET parses_channel_id = ?, - parses_posted_at = CASE WHEN ? IS NULL THEN parses_posted_at ELSE strftime('%s','now') END, - updated_at = strftime('%s','now') - WHERE discord_guild_id = ?; + SET parses_channel_id = %s, + parses_posted_at = CASE WHEN %s::text IS NULL THEN parses_posted_at ELSE floor(extract(epoch from now())) END, + updated_at = floor(extract(epoch from now())) + WHERE discord_guild_id = %s; -- :name set_parses_posted_at -UPDATE discord_guild_links SET parses_posted_at = ? WHERE discord_guild_id = ?; +UPDATE discord_guild_links SET parses_posted_at = %s WHERE discord_guild_id = %s; -- Every link with parse posting on — the parse poster's work list. -- :name select_parse_links diff --git a/backend/server/db/downloads.py b/backend/server/db/downloads.py index 5488beba..6a202baa 100644 --- a/backend/server/db/downloads.py +++ b/backend/server/db/downloads.py @@ -1,4 +1,4 @@ -"""users.db download_events helpers (async aiosqlite). +"""users download_events helpers (async psycopg). Records which signed-in users clicked each direct-download link on the Downloads page (parser installer / portable zip / ACT plugin dll). The whole @@ -12,22 +12,19 @@ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class DownloadsStore(AsyncStoreBase): - """users.db `download_events` domain. Schema is owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class DownloadsStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def record_download(self, discord_id: str, slug: str) -> bool: """Record that a user clicked a download. Idempotent per (user, slug): @@ -40,15 +37,15 @@ async def record_download(self, discord_id: str, slug: str) -> bool: async def count_for_slug(self, slug: str) -> int: """Distinct downloaders for one slug.""" async with self._db() as db: - async with db.execute(_SQL["count_for_slug"], (slug,)) as cur: + async with await db.execute(_SQL["count_for_slug"], (slug,)) as cur: row = await cur.fetchone() - return row[0] if row else 0 + return row["n"] if row else 0 async def counts(self) -> dict[str, int]: """Distinct-downloader count for every slug that has at least one row.""" async with self._db() as db: - async with db.execute(_SQL["count_all"]) as cur: - return {row[0]: row[1] for row in await cur.fetchall()} + async with await db.execute(_SQL["count_all"]) as cur: + return {row["slug"]: row["n"] for row in await cur.fetchall()} # The shared default instance — every runtime consumer goes through this. diff --git a/backend/server/db/downloads.sql b/backend/server/db/downloads.sql index a518571e..0ee70590 100644 --- a/backend/server/db/downloads.sql +++ b/backend/server/db/downloads.sql @@ -1,14 +1,14 @@ --- SQL for backend/server/db/downloads.py (async aiosqlite). +-- SQL for backend/server/db/downloads.py (psycopg, users schema). -- Record a download click. UNIQUE(discord_id, slug) makes this idempotent per -- user, so the public count is distinct-downloaders and can't be inflated by -- one person re-clicking. rowcount 0 ⇒ this user already recorded this slug. -- :name insert_download -INSERT INTO download_events (discord_id, slug) VALUES (?, ?) +INSERT INTO download_events (discord_id, slug) VALUES (%s, %s) ON CONFLICT DO NOTHING; -- :name count_for_slug -SELECT COUNT(*) FROM download_events WHERE slug = ?; +SELECT COUNT(*) AS n FROM download_events WHERE slug = %s; -- :name count_all SELECT slug, COUNT(*) AS n FROM download_events GROUP BY slug; diff --git a/backend/server/db/favorites.py b/backend/server/db/favorites.py index cbe3a3d0..22dc08aa 100644 --- a/backend/server/db/favorites.py +++ b/backend/server/db/favorites.py @@ -1,4 +1,4 @@ -"""users.db character_favorites helpers (async aiosqlite). +"""users character_favorites helpers (async psycopg). A favourite is a per-user bookmark of a (character_name, world) pair — NOT ownership; it carries no guild or claim implications. Mirrors the raid_schedule @@ -9,22 +9,19 @@ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class FavoritesStore(AsyncStoreBase): - """users.db `favorites` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class FavoritesStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def add_favorite(self, discord_id: str, character_name: str, world: str, cap: int) -> bool: """Add a favourite, atomically enforcing the per-user-per-world ``cap``. @@ -53,27 +50,27 @@ async def count_favorites_for_character(self, character_name: str, world: str) - membership lookup so the API layer can cache the count and genuinely skip this query (and its connection) on a cache hit.""" async with self._db() as db: - async with db.execute(_SQL["count_for_character"], (character_name, world)) as cur: + async with await db.execute(_SQL["count_for_character"], (character_name, world)) as cur: row = await cur.fetchone() - return row[0] if row else 0 + return row["n"] if row else 0 async def is_favorited(self, discord_id: str, character_name: str, world: str) -> bool: """Point lookup on the UNIQUE index — has this user favourited this character?""" async with self._db() as db: - async with db.execute(_SQL["select_is_favorited"], (discord_id, character_name, world)) as cur: + async with await db.execute(_SQL["select_is_favorited"], (discord_id, character_name, world)) as cur: return await cur.fetchone() is not None async def count_user_favorites(self, discord_id: str, world: str) -> int: """How many characters this user has favourited on this world.""" async with self._db() as db: - async with db.execute(_SQL["count_for_user"], (discord_id, world)) as cur: + async with await db.execute(_SQL["count_for_user"], (discord_id, world)) as cur: row = await cur.fetchone() - return row[0] if row else 0 + return row["n"] if row else 0 async def list_favorites(self, discord_id: str, world: str) -> list[dict]: """The user's favourites on this world, newest first.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_for_user"], (discord_id, world)) as cur: + async with await db.execute(_SQL["select_for_user"], (discord_id, world)) as cur: return [dict(r) for r in await cur.fetchall()] diff --git a/backend/server/db/favorites.sql b/backend/server/db/favorites.sql index d21f75df..965f014f 100644 --- a/backend/server/db/favorites.sql +++ b/backend/server/db/favorites.sql @@ -1,27 +1,29 @@ --- SQL for backend/server/db/favorites.py (async aiosqlite). +-- SQL for backend/server/db/favorites.py (psycopg, users schema). --- Cap-guarded insert: atomic within SQLite, so concurrent PUTs cannot race the --- check-then-insert past the per-world cap. Rowcount 0 means EITHER the row --- already existed (OR IGNORE) or the cap was hit — callers disambiguate with --- select_is_favorited. +-- Cap-guarded insert: the count check rides in the INSERT itself (no route- +-- layer check-then-insert window). NOTE: under Postgres read-committed two +-- concurrent inserts can each see count < cap and land cap+1 — SQLite's +-- single-writer made this fully atomic; acceptable drift for a soft bookmark +-- cap. Rowcount 0 means EITHER the row already existed (ON CONFLICT DO +-- NOTHING) or the cap was hit — callers disambiguate with select_is_favorited. -- :name insert_favorite_capped INSERT INTO character_favorites (discord_id, character_name, world) -SELECT ?, ?, ? -WHERE (SELECT COUNT(*) FROM character_favorites WHERE discord_id = ? AND world = ?) < ? +SELECT %s, %s, %s +WHERE (SELECT COUNT(*) FROM character_favorites WHERE discord_id = %s AND world = %s) < %s ON CONFLICT DO NOTHING; -- :name delete_favorite -DELETE FROM character_favorites WHERE discord_id = ? AND character_name = ? AND world = ?; +DELETE FROM character_favorites WHERE discord_id = %s AND character_name = %s AND world = %s; -- :name select_is_favorited -SELECT 1 FROM character_favorites WHERE discord_id = ? AND character_name = ? AND world = ?; +SELECT 1 FROM character_favorites WHERE discord_id = %s AND character_name = %s AND world = %s; -- :name count_for_character -SELECT COUNT(*) FROM character_favorites WHERE character_name = ? AND world = ?; +SELECT COUNT(*) AS n FROM character_favorites WHERE character_name = %s AND world = %s; -- :name count_for_user -SELECT COUNT(*) FROM character_favorites WHERE discord_id = ? AND world = ?; +SELECT COUNT(*) AS n FROM character_favorites WHERE discord_id = %s AND world = %s; -- :name select_for_user SELECT character_name, world, created_at FROM character_favorites -WHERE discord_id = ? AND world = ? ORDER BY created_at DESC, id DESC; +WHERE discord_id = %s AND world = %s ORDER BY created_at DESC, id DESC; diff --git a/backend/server/db/guild_recruitment.py b/backend/server/db/guild_recruitment.py index 9a6afe61..cc2339d7 100644 --- a/backend/server/db/guild_recruitment.py +++ b/backend/server/db/guild_recruitment.py @@ -1,4 +1,4 @@ -"""users.db ``guild_recruitment`` domain (async aiosqlite). +"""users ``guild_recruitment`` domain (async psycopg). Per-guild recruitment profiles: officer-editable, publicly viewable. Rows are keyed by the CENSUS GUILD ID so a profile (and its logo) survives a @@ -6,9 +6,9 @@ officer save and by the daily census sweep, which also auto-delists guilds whose id no longer exists in census. An absent row reads as "not recruiting" with an empty profile, so callers never merge defaults. The -200x200-max WebP logo lives in a BLOB column — users.db is -litestream-replicated, so logos survive volume wipes and are transactional -with their attribution columns. +200x200-max WebP logo lives in a bytea column — transactional with its +attribution columns and inside the same Postgres backups as everything +else. Audit/erasure notes: ``updated_by`` and ``logo_uploaded_by`` are Discord ids and are tombstoned by the erasure sweep; the logo blob itself is the @@ -19,10 +19,9 @@ from __future__ import annotations import json -from pathlib import Path -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) @@ -51,23 +50,23 @@ def _loads_list(raw: str | None) -> list[str]: return [str(v) for v in value] if isinstance(value, list) else [] -class GuildRecruitmentStore(AsyncStoreBase): - """Schema/migrations are owned by the package orchestrator - (backend.server.db.init_db); methods open per-call connections against - ``self.path``. The logo BLOB is only ever read by :meth:`get_logo` — - profile/list reads carry a ``has_logo`` flag instead. Every write that - (re)binds a name to an id first purges rows holding that name under a - different id (census name ownership is unique per world, so such rows - are provably stale — a disbanded guild whose name was recycled).""" +class GuildRecruitmentStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``. The logo bytea is + only ever read by :meth:`get_logo` — profile/list reads carry a + ``has_logo`` flag instead. Every write that (re)binds a name to an id + first purges rows holding that name under a different id (census name + ownership is unique per world, so such rows are provably stale — a + disbanded guild whose name was recycled).""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def get_profile(self, world: str, guild_name: str) -> dict: """The guild's profile with JSON columns decoded; the empty-profile defaults when the guild has no row.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_profile"], (world, guild_name)) as cur: + async with await db.execute(_SQL["select_profile"], (world, guild_name)) as cur: row = await cur.fetchone() if row is None: return dict(_EMPTY_PROFILE) @@ -127,12 +126,14 @@ async def get_logo(self, world: str, guild_name: str) -> dict | None: """``{logo, logo_media_type, logo_uploaded_at}`` or ``None`` when the guild has no row or no logo.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_logo"], (world, guild_name)) as cur: + async with await db.execute(_SQL["select_logo"], (world, guild_name)) as cur: row = await cur.fetchone() if row is None: return None return { - "logo": row["logo"], + # psycopg returns bytea as memoryview — materialise to bytes at + # the store boundary so Response(content=...) keeps working. + "logo": bytes(row["logo"]), "logo_media_type": row["logo_media_type"], "logo_uploaded_at": row["logo_uploaded_at"], } @@ -162,7 +163,7 @@ async def list_recruiting(self, world: str) -> list[dict]: """Every recruiting guild in the world (newest edit first), JSON decoded, no blobs.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_recruiting"], (world,)) as cur: + async with await db.execute(_SQL["select_recruiting"], (world,)) as cur: rows = await cur.fetchall() return [ { @@ -185,15 +186,15 @@ async def list_recruiting(self, world: str) -> list[dict]: async def list_listed_worlds(self) -> list[str]: """Every world with at least one recruiting=1 row.""" async with self._db() as db: - async with db.execute(_SQL["select_listed_worlds"]) as cur: + async with await db.execute(_SQL["select_listed_worlds"]) as cur: rows = await cur.fetchall() - return [row[0] for row in rows] + return [row["world"] for row in rows] async def list_listed_ids(self, world: str) -> list[dict]: """``[{guild_id, guild_name}]`` for every row with recruiting=1 — the sweep's work list.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_listed_ids"], (world,)) as cur: + async with await db.execute(_SQL["select_listed_ids"], (world,)) as cur: rows = await cur.fetchall() return [{"guild_id": row["guild_id"], "guild_name": row["guild_name"]} for row in rows] diff --git a/backend/server/db/guild_recruitment.sql b/backend/server/db/guild_recruitment.sql index 66851f97..baf627ed 100644 --- a/backend/server/db/guild_recruitment.sql +++ b/backend/server/db/guild_recruitment.sql @@ -1,4 +1,4 @@ --- SQL for backend/server/db/guild_recruitment.py (async aiosqlite). +-- SQL for backend/server/db/guild_recruitment.py (psycopg, users schema). -- -- Rows are KEYED by census guild id (rename-proof); guild_name is a -- lookup/display column kept fresh by officer saves and the daily census @@ -9,15 +9,15 @@ -- satisfiable. -- :name select_profile --- Explicit columns only — the logo BLOB is never selected here; readers +-- Explicit columns only — the logo bytea is never selected here; readers -- get a has_logo flag instead. SELECT guild_id, guild_name, recruiting, description, classes_json, tags_json, contacts_json, discord_url, updated_by, updated_at, (logo IS NOT NULL) AS has_logo, logo_uploaded_by, logo_uploaded_at -FROM guild_recruitment WHERE world = ? AND guild_name = ?; +FROM guild_recruitment WHERE world = %s AND guild_name = %s; -- :name purge_name_conflicts -DELETE FROM guild_recruitment WHERE world = ? AND guild_name = ? AND guild_id <> ?; +DELETE FROM guild_recruitment WHERE world = %s AND guild_name = %s AND guild_id <> %s; -- :name upsert_profile -- Profile columns ONLY — the logo columns are owned by upsert_logo and @@ -27,7 +27,7 @@ INSERT INTO guild_recruitment ( world, guild_id, guild_name, recruiting, description, classes_json, tags_json, contacts_json, discord_url, updated_by, updated_at ) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%s','now')) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, floor(extract(epoch from now()))) ON CONFLICT(world, guild_id) DO UPDATE SET guild_name = excluded.guild_name, recruiting = excluded.recruiting, @@ -37,12 +37,12 @@ ON CONFLICT(world, guild_id) DO UPDATE SET contacts_json = excluded.contacts_json, discord_url = excluded.discord_url, updated_by = excluded.updated_by, - updated_at = strftime('%s','now'); + updated_at = floor(extract(epoch from now())); -- :name select_logo SELECT logo, logo_media_type, logo_uploaded_at FROM guild_recruitment -WHERE world = ? AND guild_name = ? AND logo IS NOT NULL; +WHERE world = %s AND guild_name = %s AND logo IS NOT NULL; -- :name upsert_logo -- Logo columns ONLY — an insert on a guild with no profile row leaves the @@ -50,25 +50,25 @@ WHERE world = ? AND guild_name = ? AND logo IS NOT NULL; INSERT INTO guild_recruitment ( world, guild_id, guild_name, logo, logo_media_type, logo_uploaded_by, logo_uploaded_at ) -VALUES (?, ?, ?, ?, ?, ?, strftime('%s','now')) +VALUES (%s, %s, %s, %s, %s, %s, floor(extract(epoch from now()))) ON CONFLICT(world, guild_id) DO UPDATE SET guild_name = excluded.guild_name, logo = excluded.logo, logo_media_type = excluded.logo_media_type, logo_uploaded_by = excluded.logo_uploaded_by, - logo_uploaded_at = strftime('%s','now'); + logo_uploaded_at = floor(extract(epoch from now())); -- :name clear_logo UPDATE guild_recruitment SET logo = NULL, logo_media_type = NULL, logo_uploaded_by = NULL, logo_uploaded_at = NULL -WHERE world = ? AND guild_name = ? AND logo IS NOT NULL; +WHERE world = %s AND guild_name = %s AND logo IS NOT NULL; -- :name select_recruiting SELECT guild_id, guild_name, description, classes_json, tags_json, contacts_json, discord_url, updated_at, (logo IS NOT NULL) AS has_logo, logo_uploaded_at FROM guild_recruitment -WHERE world = ? AND recruiting = 1 +WHERE world = %s AND recruiting = 1 ORDER BY updated_at DESC; -- Sweep support: every listed row's identity for the daily census @@ -77,10 +77,10 @@ ORDER BY updated_at DESC; SELECT DISTINCT world FROM guild_recruitment WHERE recruiting = 1; -- :name select_listed_ids -SELECT guild_id, guild_name FROM guild_recruitment WHERE world = ? AND recruiting = 1; +SELECT guild_id, guild_name FROM guild_recruitment WHERE world = %s AND recruiting = 1; -- :name update_guild_name -UPDATE guild_recruitment SET guild_name = ? WHERE world = ? AND guild_id = ? AND guild_name <> ?; +UPDATE guild_recruitment SET guild_name = %s WHERE world = %s AND guild_id = %s AND guild_name <> %s; -- :name delist_guild -UPDATE guild_recruitment SET recruiting = 0 WHERE world = ? AND guild_id = ? AND recruiting = 1; +UPDATE guild_recruitment SET recruiting = 0 WHERE world = %s AND guild_id = %s AND recruiting = 1; diff --git a/backend/server/db/guild_settings.py b/backend/server/db/guild_settings.py index 3c3e5a02..9bcf1ad2 100644 --- a/backend/server/db/guild_settings.py +++ b/backend/server/db/guild_settings.py @@ -1,4 +1,4 @@ -"""users.db `guild_settings` domain (async aiosqlite). +"""users `guild_settings` domain (async psycopg). Per-guild feature switches that only the guild LEADER (Census rank_id 0) or a site admin may change. One boolean column per setting: the DB default @@ -14,10 +14,9 @@ from __future__ import annotations from collections.abc import Iterable -from pathlib import Path -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) @@ -25,19 +24,18 @@ DEFAULT_GUILD_SETTINGS: dict[str, bool] = {"officers_can_delete_parses": True} -class GuildSettingsStore(AsyncStoreBase): - """Schema/migrations are owned by the package orchestrator - (backend.server.db.init_db); methods open per-call connections against - ``self.path``.""" +class GuildSettingsStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def get_settings(self, world: str, guild_name: str) -> dict: """``{officers_can_delete_parses, updated_by, updated_at}`` — the defaults (and ``None`` audit fields) when the guild has no row.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_settings"], (world, guild_name)) as cur: + async with await db.execute(_SQL["select_settings"], (world, guild_name)) as cur: row = await cur.fetchone() if row is None: return {**DEFAULT_GUILD_SETTINGS, "updated_by": None, "updated_at": None} @@ -68,10 +66,8 @@ async def officers_can_delete_parses(self, world: str, guild_names: Iterable[str if not names: return {} flags = dict.fromkeys(names, True) - placeholders = ",".join("?" * len(names)) - sql = _SQL["select_delete_flags"].format(placeholders=placeholders) async with self._db(row_factory=True) as db: - async with db.execute(sql, (world, *names)) as cur: + async with await db.execute(_SQL["select_delete_flags"], (world, names)) as cur: for row in await cur.fetchall(): flags[row["guild_name"]] = bool(row["officers_can_delete_parses"]) return flags diff --git a/backend/server/db/guild_settings.sql b/backend/server/db/guild_settings.sql index b7d5a042..1ea10c7b 100644 --- a/backend/server/db/guild_settings.sql +++ b/backend/server/db/guild_settings.sql @@ -1,18 +1,19 @@ --- SQL for backend/server/db/guild_settings.py (async aiosqlite). +-- SQL for backend/server/db/guild_settings.py (psycopg, users schema). -- :name select_settings SELECT officers_can_delete_parses, updated_by, updated_at -FROM guild_settings WHERE world = ? AND guild_name = ?; +FROM guild_settings WHERE world = %s AND guild_name = %s; -- :name upsert_settings INSERT INTO guild_settings (world, guild_name, officers_can_delete_parses, updated_by, updated_at) -VALUES (?, ?, ?, ?, strftime('%s','now')) +VALUES (%s, %s, %s, %s, floor(extract(epoch from now()))) ON CONFLICT(world, guild_name) DO UPDATE SET officers_can_delete_parses = excluded.officers_can_delete_parses, updated_by = excluded.updated_by, - updated_at = strftime('%s','now'); + updated_at = floor(extract(epoch from now())); -- :name select_delete_flags --- {placeholders} = "?,?,…" composed in Python for the guild-name IN list. +-- The name list binds as ONE array parameter (= ANY) — no composed +-- placeholder strings and no variable-count limits. SELECT guild_name, officers_can_delete_parses -FROM guild_settings WHERE world = ? AND guild_name IN ({placeholders}); +FROM guild_settings WHERE world = %s AND guild_name = ANY(%s); diff --git a/backend/server/db/item_watch.py b/backend/server/db/item_watch.py index 29b73eb3..5d744678 100644 --- a/backend/server/db/item_watch.py +++ b/backend/server/db/item_watch.py @@ -1,28 +1,28 @@ -"""users.db item_watch table helpers. +"""users-schema item_watch table helpers (psycopg). -Carved out of the original 1309-line web/db.py. Async (aiosqlite) helpers -for the item watch domain. Per-call connections open via the -shared ``AsyncStoreBase._db()``; tests re-point ``store.path``. +Carved out of the original 1309-line web/db.py. Async helpers for the item +watch domain. Methods check out pooled connections via the shared +``PgStoreBase._db()``; tests re-point ``store.schema``. """ from __future__ import annotations -from pathlib import Path +from psycopg.errors import UniqueViolation -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class ItemWatchStore(AsyncStoreBase): - """users.db `item_watch` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class ItemWatchStore(PgStoreBase): + """users-schema `item_watch` domain. Schema DDL is owned by + db/migrations/0001_users.sql; methods check out pooled connections + scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def add_item_watch( self, @@ -45,14 +45,12 @@ async def add_item_watch( _SQL["add_watch"], (world, guild_name, character_name, item_id, item_name, added_by, added_by_name), ) - except Exception as exc: - if "UNIQUE" in str(exc): - raise ValueError(f"'{item_name}' is already being watched for {character_name}.") from exc - raise + except UniqueViolation as exc: + raise ValueError(f"'{item_name}' is already being watched for {character_name}.") from exc _row = await cur.fetchone() - new_id = _row[0] if _row else None + new_id = _row["id"] if _row else None await db.commit() - async with db.execute(_SQL["find_by_id"], (new_id,)) as cur2: + async with await db.execute(_SQL["find_by_id"], (new_id,)) as cur2: row = await cur2.fetchone() assert row is not None, "INSERT succeeded but SELECT returned nothing" return dict(row) @@ -64,7 +62,7 @@ async def list_item_watches( ) -> list[dict]: """Return all item watch entries for a guild on a given server, ordered by added_at descending.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["list_for_guild"], (guild_name, world)) as cur: + async with await db.execute(_SQL["list_for_guild"], (guild_name, world)) as cur: rows = await cur.fetchall() return [dict(r) for r in rows] diff --git a/backend/server/db/item_watch.sql b/backend/server/db/item_watch.sql index bd96c6ec..c4ce0fdd 100644 --- a/backend/server/db/item_watch.sql +++ b/backend/server/db/item_watch.sql @@ -1,26 +1,26 @@ --- SQL for backend/server/db/item_watch.py (async aiosqlite). +-- SQL for backend/server/db/item_watch.py (psycopg, users schema). -- :name add_watch INSERT INTO item_watch (world, guild_name, character_name, item_id, item_name, added_by, added_by_name) -VALUES (?, ?, ?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING id; -- :name find_by_id -SELECT * FROM item_watch WHERE id = ?; +SELECT * FROM item_watch WHERE id = %s; -- :name list_for_guild -SELECT * FROM item_watch WHERE guild_name = ? AND world = ? ORDER BY added_at DESC; +SELECT * FROM item_watch WHERE guild_name = %s AND world = %s ORDER BY added_at DESC; -- :name remove_watch -DELETE FROM item_watch WHERE id = ? AND guild_name = ? AND world = ?; +DELETE FROM item_watch WHERE id = %s AND guild_name = %s AND world = %s; -- :name update_seen UPDATE item_watch SET - last_checked_at = strftime('%s','now'), - last_seen_at = strftime('%s','now'), - first_seen_at = COALESCE(first_seen_at, strftime('%s','now')) -WHERE id = ?; + last_checked_at = floor(extract(epoch from now())), + last_seen_at = floor(extract(epoch from now())), + first_seen_at = COALESCE(first_seen_at, floor(extract(epoch from now()))) +WHERE id = %s; -- :name update_unseen -UPDATE item_watch SET last_checked_at = strftime('%s','now') WHERE id = ?; +UPDATE item_watch SET last_checked_at = floor(extract(epoch from now())) WHERE id = %s; diff --git a/backend/server/db/raid_planning.py b/backend/server/db/raid_planning.py index 3a30054d..7711c45f 100644 --- a/backend/server/db/raid_planning.py +++ b/backend/server/db/raid_planning.py @@ -1,4 +1,4 @@ -"""users.db raid-planning helpers (async aiosqlite). +"""users raid-planning helpers (async psycopg). Two concerns, both officer-curated and guild-member-viewable: @@ -10,16 +10,14 @@ the raid-schedule editor regenerates team rows on save (ids are not stable); the schedule PUT prunes placements for removed teams. -Per-call connections via the shared ``AsyncStoreBase._db()``; tests -re-point ``store.path``. +Per-call pooled connections via the shared ``PgStoreBase._db()``; tests +re-point ``store.schema``. """ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) @@ -27,19 +25,19 @@ VALID_ROLES = ("raider", "raid_alt") -class RaidPlanningStore(AsyncStoreBase): - """users.db `raid_planning` domain. Schema/migrations are owned by the - package orchestrator (backend.server.db.init_db).""" +class RaidPlanningStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) # ── Roster roles ───────────────────────────────────────────────────────── async def get_roles(self, world: str, guild_name: str) -> list[dict]: """All roster designations for a guild: [{character_name, role, ...}].""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_roles"], (world, guild_name)) as cur: + async with await db.execute(_SQL["select_roles"], (world, guild_name)) as cur: return [dict(r) for r in await cur.fetchall()] async def set_role( @@ -79,7 +77,7 @@ async def set_role( async def get_placements(self, world: str, guild_name: str, team_index: int) -> list[dict]: """One team's layout: [{character_name, group_num, slot, sitout}].""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_placements"], (world, guild_name, team_index)) as cur: + async with await db.execute(_SQL["select_placements"], (world, guild_name, team_index)) as cur: return [dict(r) for r in await cur.fetchall()] async def replace_placements( @@ -98,7 +96,8 @@ async def replace_placements( """ async with self._db() as db: try: - await db.execute("BEGIN") + # psycopg opens the transaction implicitly at the first + # execute — no explicit BEGIN (it would warn on the server). await db.execute(_SQL["delete_placements_for_team"], (world, guild_name, team_index)) for p in placements: await db.execute( @@ -131,7 +130,7 @@ async def roles_for_world(self, world: str) -> dict[str, str]: the home-page availability panel checks the viewer's claimed characters against this to decide whether to show itself.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_roles_for_world"], (world,)) as cur: + async with await db.execute(_SQL["select_roles_for_world"], (world,)) as cur: return {r["name_lower"]: r["role"] for r in await cur.fetchall()} # ── Player mapping ─────────────────────────────────────────────────────── @@ -141,14 +140,14 @@ async def claims_map(self, world: str) -> dict[str, str]: world — who plays whom, for availability overlay + the duplicate- player warning.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_claims_for_world"], (world,)) as cur: + async with await db.execute(_SQL["select_claims_for_world"], (world,)) as cur: return {r["name_lower"]: r["discord_id"] for r in await cur.fetchall()} async def primary_claims(self, world: str) -> set[str]: """Lower-cased names of every primary-claimed character on the world — the tie-breaker when a player has several rostered raiders.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_primary_claims_for_world"], (world,)) as cur: + async with await db.execute(_SQL["select_primary_claims_for_world"], (world,)) as cur: return {r["name_lower"] for r in await cur.fetchall()} diff --git a/backend/server/db/raid_planning.sql b/backend/server/db/raid_planning.sql index f4005eeb..809fb47c 100644 --- a/backend/server/db/raid_planning.sql +++ b/backend/server/db/raid_planning.sql @@ -1,9 +1,9 @@ --- SQL for backend/server/db/raid_planning.py (RaidPlanningStore). +-- SQL for backend/server/db/raid_planning.py (psycopg, users schema). -- :name select_roles SELECT character_name, role, placeholder, cls, updated_at, updated_by FROM raid_roster_roles -WHERE world = ? AND guild_name = ? +WHERE world = %s AND guild_name = %s ORDER BY LOWER(character_name); -- A normal (placeholder=0) write for the same name supersedes an earlier @@ -11,7 +11,7 @@ ORDER BY LOWER(character_name); -- hand-added stand-in. -- :name upsert_role INSERT INTO raid_roster_roles (world, guild_name, character_name, role, placeholder, cls, updated_at, updated_by) -VALUES (?, ?, ?, ?, ?, ?, strftime('%s','now'), ?) +VALUES (%s, %s, %s, %s, %s, %s, floor(extract(epoch from now())), %s) ON CONFLICT(world, guild_name, character_name) DO UPDATE SET role = excluded.role, placeholder = excluded.placeholder, @@ -21,30 +21,32 @@ ON CONFLICT(world, guild_name, character_name) DO UPDATE SET -- :name delete_role DELETE FROM raid_roster_roles -WHERE world = ? AND guild_name = ? AND LOWER(character_name) = LOWER(?); +WHERE world = %s AND guild_name = %s AND LOWER(character_name) = LOWER(%s); -- :name delete_placements_for_character DELETE FROM raid_placements -WHERE world = ? AND guild_name = ? AND LOWER(character_name) = LOWER(?); +WHERE world = %s AND guild_name = %s AND LOWER(character_name) = LOWER(%s); -- :name select_placements +-- NULLS FIRST preserves the SQLite ordering (benched rows with NULL +-- group_num/slot sorted before placed ones); Postgres defaults NULLS LAST. SELECT character_name, group_num, slot, sitout FROM raid_placements -WHERE world = ? AND guild_name = ? AND team_index = ? -ORDER BY group_num, slot, LOWER(character_name); +WHERE world = %s AND guild_name = %s AND team_index = %s +ORDER BY group_num NULLS FIRST, slot NULLS FIRST, LOWER(character_name); -- :name delete_placements_for_team DELETE FROM raid_placements -WHERE world = ? AND guild_name = ? AND team_index = ?; +WHERE world = %s AND guild_name = %s AND team_index = %s; -- :name insert_placement INSERT INTO raid_placements (world, guild_name, team_index, character_name, group_num, slot, sitout, updated_at, updated_by) -VALUES (?, ?, ?, ?, ?, ?, ?, strftime('%s','now'), ?); +VALUES (%s, %s, %s, %s, %s, %s, %s, floor(extract(epoch from now())), %s); -- :name prune_placements_beyond DELETE FROM raid_placements -WHERE world = ? AND guild_name = ? AND team_index >= ?; +WHERE world = %s AND guild_name = %s AND team_index >= %s; -- Approved claims for a guild's characters: who plays whom. Availability and -- the duplicate-player warning both key off this. Case-insensitive join side @@ -52,14 +54,14 @@ WHERE world = ? AND guild_name = ? AND team_index >= ?; -- :name select_claims_for_world SELECT LOWER(character_name) AS name_lower, discord_id FROM character_claims -WHERE world = ? AND status = 'approved'; +WHERE world = %s AND status = 'approved'; -- Primary-claim flags for main resolution (attendance "raid main" -- attribution): which of a player's claims is their designated primary. -- :name select_primary_claims_for_world SELECT LOWER(character_name) AS name_lower FROM character_claims -WHERE world = ? AND status = 'approved' AND is_primary = 1; +WHERE world = %s AND status = 'approved' AND is_primary = 1; -- Is any of these characters on a raid roster anywhere on this world? -- Drives the home-page availability panel's is_raider gate. Name matching is @@ -67,4 +69,4 @@ WHERE world = ? AND status = 'approved' AND is_primary = 1; -- :name select_roles_for_world SELECT LOWER(character_name) AS name_lower, role FROM raid_roster_roles -WHERE world = ?; +WHERE world = %s; diff --git a/backend/server/db/raid_schedule.py b/backend/server/db/raid_schedule.py index 2e9a1c87..e096c93a 100644 --- a/backend/server/db/raid_schedule.py +++ b/backend/server/db/raid_schedule.py @@ -1,43 +1,40 @@ -"""users.db raid_teams / raid_slots helpers (async aiosqlite). +"""users-schema raid_teams / raid_slots helpers (psycopg). -Officer-editable, publicly-viewable guild raid schedules. Mirrors the -item_watch domain: per-call connections via the shared ``AsyncStoreBase._db()``; -tests re-point ``store.path``. A team carries a ``raids`` list of its slots. +Officer-editable, publicly-viewable guild raid schedules. A team carries a +``raids`` list of its slots; ``days`` is a real integer[] column (ISO +weekdays) — no CSV round-trip anywhere. """ from __future__ import annotations -from pathlib import Path from typing import Any -import aiosqlite - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class RaidScheduleStore(AsyncStoreBase): - """users.db `raid_schedule` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class RaidScheduleStore(PgStoreBase): + """users-schema `raid_schedule` domain. Schema DDL is owned by + db/migrations/0001_users.sql; methods check out pooled connections + scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) @staticmethod - async def _teams_with_slots(db: aiosqlite.Connection, teams: list[dict]) -> list[dict]: + async def _teams_with_slots(db: Any, teams: list[dict]) -> list[dict]: for t in teams: - async with db.execute(_SQL["select_slots"], (t["id"],)) as cur: + async with await db.execute(_SQL["select_slots"], (t["id"],)) as cur: t["raids"] = [dict(r) for r in await cur.fetchall()] return teams async def get_schedule(self, world: str, guild_name: str) -> list[dict]: """Return this guild's raid teams (ordered) each with a ``raids`` list.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_teams"], (world, guild_name)) as cur: + async with await db.execute(_SQL["select_teams"], (world, guild_name)) as cur: teams = [dict(r) for r in await cur.fetchall()] return await RaidScheduleStore._teams_with_slots(db, teams) @@ -57,7 +54,7 @@ async def replace_schedule( """ async with self._db() as db: try: - await db.execute("BEGIN") + # psycopg opens the transaction implicitly on first execute. await db.execute(_SQL["delete_slots_for_guild"], (world, guild_name)) await db.execute(_SQL["delete_teams_for_guild"], (world, guild_name)) for team_index, team in enumerate(teams): @@ -74,7 +71,7 @@ async def replace_schedule( ), ) _row = await cur.fetchone() - team_id = _row[0] if _row else None + team_id = _row["id"] if _row else None for slot_index, raid in enumerate(team.get("raids", [])): await db.execute( _SQL["insert_slot"], @@ -96,7 +93,7 @@ async def list_all_teams_with_twitch(self) -> list[dict]: """Every team across all worlds/guilds that has a twitch_login, each with its raids. Used by the Twitch-live poller (Part 2).""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_teams_with_twitch"]) as cur: + async with await db.execute(_SQL["select_teams_with_twitch"]) as cur: teams = [dict(r) for r in await cur.fetchall()] return await RaidScheduleStore._teams_with_slots(db, teams) diff --git a/backend/server/db/raid_schedule.sql b/backend/server/db/raid_schedule.sql index 4e5a6997..e25dedf9 100644 --- a/backend/server/db/raid_schedule.sql +++ b/backend/server/db/raid_schedule.sql @@ -1,10 +1,10 @@ --- SQL for backend/server/db/raid_schedule.py (async aiosqlite). +-- SQL for backend/server/db/raid_schedule.py (psycopg, users schema). -- :name select_teams -SELECT * FROM raid_teams WHERE world = ? AND guild_name = ? ORDER BY team_index; +SELECT * FROM raid_teams WHERE world = %s AND guild_name = %s ORDER BY team_index; -- :name select_slots -SELECT * FROM raid_slots WHERE team_id = ? ORDER BY slot_index; +SELECT * FROM raid_slots WHERE team_id = %s ORDER BY slot_index; -- :name select_teams_with_twitch SELECT * FROM raid_teams WHERE twitch_login IS NOT NULL AND twitch_login <> ''; @@ -13,17 +13,17 @@ SELECT * FROM raid_teams WHERE twitch_login IS NOT NULL AND twitch_login <> ''; -- depend on ON DELETE CASCADE being enabled on the runtime connection. -- :name delete_slots_for_guild DELETE FROM raid_slots WHERE team_id IN ( - SELECT id FROM raid_teams WHERE world = ? AND guild_name = ? + SELECT id FROM raid_teams WHERE world = %s AND guild_name = %s ); -- :name delete_teams_for_guild -DELETE FROM raid_teams WHERE world = ? AND guild_name = ?; +DELETE FROM raid_teams WHERE world = %s AND guild_name = %s; -- :name insert_team INSERT INTO raid_teams (world, guild_name, team_index, name, primary_tz, twitch_login, updated_by) -VALUES (?, ?, ?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING id; -- :name insert_slot INSERT INTO raid_slots (team_id, slot_index, days, start_min, end_min, label) -VALUES (?, ?, ?, ?, ?, ?); +VALUES (%s, %s, %s, %s, %s, %s); diff --git a/backend/server/db/servers.py b/backend/server/db/servers.py index 358a8a93..ff76f80f 100644 --- a/backend/server/db/servers.py +++ b/backend/server/db/servers.py @@ -1,34 +1,34 @@ -"""users.db servers table helpers (per-server registry). +"""users-schema servers table helpers (per-server registry). -Carved out of the original 1309-line web/db.py. Synchronous (sqlite3) helpers -for the servers domain — these are called at startup and from sync admin -endpoints, so they use the stdlib driver directly. +Synchronous psycopg helpers — called at startup (load_registry) and from +sync admin paths, through backend.pg's sync pool (direct connections +before the lifespan opens it). -Tests re-point ``store.path`` or construct ``ServersStore(tmp_db)``. +Tests re-point ``store.schema`` or construct ``ServersStore(schema)``. """ from __future__ import annotations -import sqlite3 -from pathlib import Path +from typing import Any -from backend.db_catalogue import PathBound -from backend.server.db import DB_PATH +from backend import pg +from backend.db_catalogue import SchemaBound +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class ServersStore(PathBound): - """users.db `servers` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class ServersStore(SchemaBound): + """users-schema `servers` domain (the one synchronous store — it must + never grow async methods; startup registry loading is sync). Schema DDL + is owned by db/migrations/0001_users.sql.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) @staticmethod - def _server_row(row: sqlite3.Row) -> dict: + def _server_row(row: dict[str, Any]) -> dict: return { "world": row["world"], "subdomain": row["subdomain"], @@ -43,19 +43,19 @@ def _server_row(row: sqlite3.Row) -> dict: } def list_servers_sync(self) -> list[dict]: - with sqlite3.connect(self.path) as conn: - conn.row_factory = sqlite3.Row - return [ServersStore._server_row(r) for r in conn.execute(_SQL["list_all"])] + with pg.connection() as conn: + conn.execute(pg.search_path_sql(self.schema)) + return [ServersStore._server_row(r) for r in conn.execute(_SQL["list_all"]).fetchall()] def get_server_by_subdomain_sync(self, subdomain: str) -> dict | None: - with sqlite3.connect(self.path) as conn: - conn.row_factory = sqlite3.Row + with pg.connection() as conn: + conn.execute(pg.search_path_sql(self.schema)) row = conn.execute(_SQL["find_by_subdomain"], (subdomain.lower(),)).fetchone() return ServersStore._server_row(row) if row else None def get_server_by_world_sync(self, world: str) -> dict | None: - with sqlite3.connect(self.path) as conn: - conn.row_factory = sqlite3.Row + with pg.connection() as conn: + conn.execute(pg.search_path_sql(self.schema)) row = conn.execute(_SQL["find_by_world"], (world,)).fetchone() return ServersStore._server_row(row) if row else None @@ -69,7 +69,8 @@ def upsert_server_settings_sync( next_xpac: str | None = None, next_xpac_dt: str | None = None, ) -> None: - with sqlite3.connect(self.path) as conn: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(self.schema)) conn.execute( _SQL["upsert_server_settings"], (max_level, current_xpac, launch_dt, next_xpac, next_xpac_dt, world), @@ -81,7 +82,8 @@ def apply_xpac_rollover_sync(self, world: str, *, current_xpac: str, max_level: the level cap, stamp the rollover instant (the rankings era-lock cutoff) and clear the countdown fields. Guarded on next_xpac still being set so a concurrent/repeat call is a no-op.""" - with sqlite3.connect(self.path) as conn: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(self.schema)) cur = conn.execute(_SQL["apply_xpac_rollover"], (current_xpac, max_level, started_dt, world)) conn.commit() return cur.rowcount > 0 @@ -93,7 +95,8 @@ def set_default_server_sync(self, world: str) -> bool: unknown (i.e. no row matched the second UPDATE — there are never 0 defaults after this call succeeds). """ - with sqlite3.connect(self.path) as conn: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(self.schema)) # First clear all, then set the target. Single transaction → never 0 or 2 defaults. conn.execute(_SQL["clear_all_defaults"]) cur = conn.execute(_SQL["set_default_by_world"], (world,)) diff --git a/backend/server/db/servers.sql b/backend/server/db/servers.sql index 8ece40a3..0eef4509 100644 --- a/backend/server/db/servers.sql +++ b/backend/server/db/servers.sql @@ -4,22 +4,22 @@ SELECT * FROM servers ORDER BY display_name; -- :name find_by_subdomain -SELECT * FROM servers WHERE subdomain = ?; +SELECT * FROM servers WHERE subdomain = %s; -- :name find_by_world -SELECT * FROM servers WHERE world = ?; +SELECT * FROM servers WHERE world = %s; -- :name upsert_server_settings -UPDATE servers SET max_level = ?, current_xpac = ?, launch_dt = ?, - next_xpac = ?, next_xpac_dt = ?, - updated_at = strftime('%s','now') -WHERE world = ?; +UPDATE servers SET max_level = %s, current_xpac = %s, launch_dt = %s, + next_xpac = %s, next_xpac_dt = %s, + updated_at = floor(extract(epoch from now())) +WHERE world = %s; -- :name clear_all_defaults UPDATE servers SET is_default = 0; -- :name set_default_by_world -UPDATE servers SET is_default = 1 WHERE world = ?; +UPDATE servers SET is_default = 1 WHERE world = %s; -- :name set_default_fallback UPDATE servers SET is_default = 1 WHERE world = @@ -29,7 +29,7 @@ UPDATE servers SET is_default = 1 WHERE world = -- still being set: a repeat/concurrent call matches zero rows. -- :name apply_xpac_rollover UPDATE servers - SET current_xpac = ?, max_level = ?, current_xpac_started_dt = ?, + SET current_xpac = %s, max_level = %s, current_xpac_started_dt = %s, next_xpac = NULL, next_xpac_dt = NULL, - updated_at = strftime('%s','now') - WHERE world = ? AND next_xpac IS NOT NULL; + updated_at = floor(extract(epoch from now())) + WHERE world = %s AND next_xpac IS NOT NULL; diff --git a/backend/server/db/site_settings.py b/backend/server/db/site_settings.py index aa94d8a8..10468d51 100644 --- a/backend/server/db/site_settings.py +++ b/backend/server/db/site_settings.py @@ -1,4 +1,4 @@ -"""users.db `site_settings` domain (async aiosqlite). +"""users-schema `site_settings` domain (psycopg). Site-wide, admin-editable string knobs — as opposed to the per-server rows in `servers`. First key: ``discord_invite_url`` (the "Join our Discord @@ -7,10 +7,8 @@ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) @@ -18,18 +16,18 @@ DISCORD_INVITE_URL_KEY = "discord_invite_url" -class SiteSettingsStore(AsyncStoreBase): - """Schema is owned by the package orchestrator (backend.server.db.init_db); - methods open per-call connections against ``self.path``.""" +class SiteSettingsStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def get_setting(self, key: str) -> str | None: async with self._db() as db: - async with db.execute(_SQL["select_setting"], (key,)) as cur: + async with await db.execute(_SQL["select_setting"], (key,)) as cur: row = await cur.fetchone() - return None if row is None else str(row[0]) + return None if row is None else str(row["value"]) async def set_setting(self, key: str, value: str | None, *, updated_by: str) -> None: """Write a setting; ``None`` or an empty string removes the row.""" @@ -42,8 +40,8 @@ async def set_setting(self, key: str, value: str | None, *, updated_by: str) -> async def all_settings(self) -> dict[str, str]: async with self._db() as db: - async with db.execute(_SQL["select_all"]) as cur: - return {str(k): str(v) for k, v in await cur.fetchall()} + async with await db.execute(_SQL["select_all"]) as cur: + return {str(r["key"]): str(r["value"]) for r in await cur.fetchall()} # The shared default instance — every runtime consumer goes through this. diff --git a/backend/server/db/site_settings.sql b/backend/server/db/site_settings.sql index b9129aa5..7dd4d513 100644 --- a/backend/server/db/site_settings.sql +++ b/backend/server/db/site_settings.sql @@ -1,18 +1,18 @@ --- SQL for backend/server/db/site_settings.py (async aiosqlite). +-- SQL for backend/server/db/site_settings.py (psycopg, users schema). -- :name select_setting -SELECT value FROM site_settings WHERE key = ?; +SELECT value FROM site_settings WHERE key = %s; -- :name select_all SELECT key, value FROM site_settings ORDER BY key; -- :name upsert_setting INSERT INTO site_settings (key, value, updated_by, updated_at) -VALUES (?, ?, ?, strftime('%s','now')) +VALUES (%s, %s, %s, floor(extract(epoch from now()))) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_by = excluded.updated_by, - updated_at = strftime('%s','now'); + updated_at = floor(extract(epoch from now())); -- :name delete_setting -DELETE FROM site_settings WHERE key = ?; +DELETE FROM site_settings WHERE key = %s; diff --git a/backend/server/db/tokens.py b/backend/server/db/tokens.py index 0737e72a..83d74435 100644 --- a/backend/server/db/tokens.py +++ b/backend/server/db/tokens.py @@ -1,8 +1,8 @@ -"""users.db api_tokens table helpers. +"""users-schema api_tokens table helpers (psycopg). -Carved out of the original 1309-line web/db.py. Async (aiosqlite) helpers -for the API token domain. Per-call connections open via the shared -``AsyncStoreBase._db()``; tests re-point ``store.path``. +Carved out of the original 1309-line web/db.py. Async helpers for the API +token domain. Methods check out pooled connections via the shared +``PgStoreBase._db()``; tests re-point ``store.schema``. Raw tokens are 'eq2c_' + 32 url-safe base64 chars (≈192 bits entropy). Only the SHA-256 hash is stored; the raw token is shown to the user once @@ -14,12 +14,12 @@ import hashlib import logging import secrets -import sqlite3 import time -from pathlib import Path -from backend.db_catalogue import AsyncStoreBase -from backend.server.db import DB_PATH +import psycopg + +from backend.db_catalogue import PgStoreBase +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _log = logging.getLogger(__name__) @@ -29,13 +29,13 @@ TOKEN_PREFIX = "eq2c_" -class TokensStore(AsyncStoreBase): - """users.db `tokens` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class TokensStore(PgStoreBase): + """users-schema `tokens` domain. Schema DDL is owned by + db/migrations/0001_users.sql; methods check out pooled connections + scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) @staticmethod def generate_token() -> tuple[str, str, str]: @@ -71,9 +71,9 @@ async def mint_api_token( (user_id, name, h, prefix), ) _row = await cur.fetchone() - new_id = _row[0] if _row else None + new_id = _row["id"] if _row else None await db.commit() - async with db.execute(_SQL["find_by_id"], (new_id,)) as cur2: + async with await db.execute(_SQL["find_by_id"], (new_id,)) as cur2: row = await cur2.fetchone() assert row is not None return raw, dict(row) @@ -81,7 +81,7 @@ async def mint_api_token( async def list_api_tokens(self, user_id: str) -> list[dict]: """All tokens for a user, newest first. Hash is omitted — UI doesn't need it.""" async with self._db(row_factory=True) as db: - async with db.execute( + async with await db.execute( _SQL["list_for_user"], (user_id,), ) as cur: @@ -111,7 +111,7 @@ async def lookup_api_token(self, raw_token: str) -> dict | None: return None h = TokensStore.hash_token(raw_token) async with self._db(row_factory=True) as db: - async with db.execute( + async with await db.execute( _SQL["lookup_by_hash"], (h,), ) as cur: @@ -121,28 +121,27 @@ async def lookup_api_token(self, raw_token: str) -> dict | None: # Coalesce last_used_at writes to 60s buckets. # # Plugin uploads fire multiple times per second during a raid; committing - # an UPDATE on every upload was a real write-storm risk (WAL mitigates - # locking but the disk write itself is the cost). Sub-minute precision - # on this column isn't useful — the UI shows "last used 5 min ago", - # not "last used 0.6 seconds ago". The existing SELECT already pulled - # the current value as part of the row fetch in lookup callers; check - # against it here. + # an UPDATE on every upload was a real write-storm risk. Sub-minute + # precision on this column isn't useful — the UI shows "last used + # 5 min ago", not "last used 0.6 seconds ago". The existing SELECT + # already pulled the current value as part of the row fetch in lookup + # callers; check against it here. now = int(time.time()) last_used = row["last_used_at"] did_write = last_used is None or (now - int(last_used)) >= 60 if did_write: # The touch is COSMETIC bookkeeping — it must never fail the - # auth path. Under a raid-night upload burst, concurrent - # writers can still contend past the busy timeout; a lost - # last-used bump costs nothing, a 500 here fails the whole - # upload (seen live 2026-09-12 on /attendance/ingest). + # auth path. A transient connection fault or lock contention + # under a raid-night upload burst costs a lost last-used bump, + # which is nothing; a 500 here fails the whole upload (seen + # live 2026-09-12 on /attendance/ingest, SQLite era). try: await db.execute( _SQL["update_last_used_at"], (now, row["token_id"]), ) await db.commit() - except sqlite3.OperationalError as exc: + except psycopg.OperationalError as exc: _log.warning("[tokens] last-used touch skipped (db busy): %s", exc) did_write = False result = dict(row) diff --git a/backend/server/db/tokens.sql b/backend/server/db/tokens.sql index a64fc5e3..aee0fe26 100644 --- a/backend/server/db/tokens.sql +++ b/backend/server/db/tokens.sql @@ -1,25 +1,25 @@ --- SQL for backend/server/db/tokens.py (async aiosqlite). +-- SQL for backend/server/db/tokens.py (psycopg, users schema). -- API tokens are hashed (sha256); raw token is shown once at mint time. -- :name mint_token INSERT INTO api_tokens (user_id, name, token_hash, token_prefix) -VALUES (?, ?, ?, ?) +VALUES (%s, %s, %s, %s) RETURNING id; -- :name find_by_id -SELECT * FROM api_tokens WHERE id = ?; +SELECT * FROM api_tokens WHERE id = %s; -- :name list_for_user -- Hash + user_id omitted from the SELECT — UI doesn't need them. SELECT id, name, token_prefix, created_at, last_used_at, revoked_at FROM api_tokens -WHERE user_id = ? +WHERE user_id = %s ORDER BY created_at DESC; -- :name revoke_token UPDATE api_tokens -SET revoked_at = strftime('%s','now') -WHERE id = ? AND user_id = ? AND revoked_at IS NULL; +SET revoked_at = floor(extract(epoch from now())) +WHERE id = %s AND user_id = %s AND revoked_at IS NULL; -- :name lookup_by_hash SELECT t.id AS token_id, t.user_id, t.name AS token_name, t.revoked_at, @@ -28,7 +28,7 @@ SELECT t.id AS token_id, t.user_id, t.name AS token_name, t.revoked_at, u.access_status FROM api_tokens t JOIN users u ON u.discord_id = t.user_id -WHERE t.token_hash = ?; +WHERE t.token_hash = %s; -- :name update_last_used_at -UPDATE api_tokens SET last_used_at = ? WHERE id = ?; +UPDATE api_tokens SET last_used_at = %s WHERE id = %s; diff --git a/backend/server/db/users.py b/backend/server/db/users.py index b024f63e..a3b40404 100644 --- a/backend/server/db/users.py +++ b/backend/server/db/users.py @@ -1,29 +1,27 @@ -"""users.db users table + role / role_request / role_permission helpers. +"""users `users` domain (async psycopg). -Carved out of the original 1309-line web/db.py. Async (aiosqlite) helpers -for the users domain. Per-call connections open via the shared -``AsyncStoreBase._db()``; tests re-point ``store.path``. +Carved out of the original 1309-line web/db.py. Async helpers for the +users + role / role_request / role_permission domain. Per-call pooled +connections via the shared ``PgStoreBase._db()``; tests re-point +``store.schema``. """ from __future__ import annotations -from pathlib import Path - -from backend.db_catalogue import AsyncStoreBase +from backend.db_catalogue import PgStoreBase from backend.server.core.sql_helpers import build_where -from backend.server.db import DB_PATH +from backend.server.db import SCHEMA from backend.sql_loader import load_sql _SQL = load_sql(__file__) -class UsersStore(AsyncStoreBase): - """users.db `users` domain. Schema/migrations are owned by the package - orchestrator (backend.server.db.init_db); methods open per-call - connections against ``self.path``.""" +class UsersStore(PgStoreBase): + """Schema DDL is owned by db/migrations/0001_users.sql; methods check + out pooled connections scoped to ``self.schema``.""" - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) async def upsert_user( self, @@ -62,14 +60,14 @@ async def upsert_user( ), ) await db.commit() - async with db.execute(_SQL["select_access_status"], (discord_id,)) as cur: + async with await db.execute(_SQL["select_access_status"], (discord_id,)) as cur: row = await cur.fetchone() return row["access_status"] if row else "pending" async def get_user_access_status(self, discord_id: str) -> str: """Return the access_status for a user, or 'pending' if not found.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["select_access_status"], (discord_id,)) as cur: + async with await db.execute(_SQL["select_access_status"], (discord_id,)) as cur: row = await cur.fetchone() return row["access_status"] if row else "pending" @@ -82,18 +80,16 @@ async def get_display_names_for_discord_ids(self, ids: list[str]) -> dict[str, s if not ids: return {} async with self._db(row_factory=True) as db: - placeholders = ",".join("?" for _ in ids) - async with db.execute( - _SQL["select_display_names_by_ids"].format(placeholders=placeholders), - ids, - ) as cur: + # The id list binds as ONE array parameter (= ANY) — no composed + # placeholder strings and no variable-count limits. + async with await db.execute(_SQL["select_display_names_by_ids"], (ids,)) as cur: rows = await cur.fetchall() return {r["discord_id"]: r["discord_name"] for r in rows} async def list_pending_users(self) -> list[dict]: """Return all users with access_status = 'pending', newest first.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["list_pending_users"]) as cur: + async with await db.execute(_SQL["list_pending_users"]) as cur: rows = await cur.fetchall() return [dict(r) for r in rows] @@ -110,7 +106,7 @@ async def approve_all_pending(self) -> int: async def list_all_users(self) -> list[dict]: """Return all users with access_status and total claim count, newest first.""" async with self._db(row_factory=True) as db: - async with db.execute(_SQL["list_all_users_with_claim_count"]) as cur: + async with await db.execute(_SQL["list_all_users_with_claim_count"]) as cur: rows = await cur.fetchall() return [dict(r) for r in rows] @@ -162,18 +158,18 @@ async def revoke_role(self, discord_id: str, role: str) -> bool: async def list_roles_for_user(self, discord_id: str) -> list[str]: """All roles assigned to a single user, sorted for stable display.""" async with self._db() as db: - async with db.execute( + async with await db.execute( _SQL["list_roles_for_user"], (discord_id,), ) as cur: rows = await cur.fetchall() - return [r[0] for r in rows] + return [r["role"] for r in rows] async def has_role(self, discord_id: str, role: str) -> bool: """Cheap (indexed) existence check — used on the hot path of the editor auth dep, so kept as a single-row SELECT rather than reusing list_roles.""" async with self._db() as db: - async with db.execute( + async with await db.execute( _SQL["check_has_role"], (discord_id, role), ) as cur: @@ -185,10 +181,10 @@ async def create_role_request( role: str, user_note: str | None, ) -> int: - """Submit a pending role request. Raises ``sqlite3.IntegrityError`` if the - user already has a pending request for this role (the partial unique index - on (discord_id, role) WHERE status='pending' enforces it). Returns the new - request's id.""" + """Submit a pending role request. Raises ``psycopg.errors.UniqueViolation`` + if the user already has a pending request for this role (the partial unique + index on (discord_id, role) WHERE status='pending' enforces it). Returns + the new request's id.""" async with self._db() as db: cur = await db.execute( _SQL["create_role_request"], @@ -196,7 +192,7 @@ async def create_role_request( ) row = await cur.fetchone() await db.commit() - return int(row[0]) if row else 0 + return int(row["id"]) if row else 0 async def list_role_requests( self, @@ -213,10 +209,10 @@ async def list_role_requests( where: list[str] = [] params: list = [] if status is not None: - where.append("rr.status = ?") + where.append("rr.status = %s") params.append(status) if discord_id is not None: - where.append("rr.discord_id = ?") + where.append("rr.discord_id = %s") params.append(discord_id) where_sql = build_where(where) # Pending: oldest first (FIFO queue). Anything resolved: newest first. @@ -226,7 +222,7 @@ async def list_role_requests( else "ORDER BY rr.requested_at DESC, rr.id DESC" ) async with self._db(row_factory=True) as db: - async with db.execute( + async with await db.execute( _SQL["list_role_requests"].format(where_sql=where_sql, order_sql=order_sql), params, ) as cur: @@ -238,7 +234,7 @@ async def get_role_request(self, request_id: int) -> dict | None: admin approve/reject endpoints for the 404 check + the role/discord_id payload that the grant flow needs.""" async with self._db(row_factory=True) as db: - async with db.execute( + async with await db.execute( _SQL["get_role_request"], (request_id,), ) as cur: @@ -289,16 +285,15 @@ async def review_and_grant_role( if cur.rowcount == 0: return None # Fetch the request row so we can grant the role - async with db.execute( + async with await db.execute( _SQL["select_role_request_grant_info"], (request_id,), ) as sel: row = await sel.fetchone() if row is not None: - discord_id, role = row await db.execute( _SQL["grant_role"], - (discord_id, role, admin_id), + (row["discord_id"], row["role"], admin_id), ) await db.commit() return await self.get_role_request(request_id) @@ -330,7 +325,7 @@ async def user_has_capability_via_db( admin (synthetic) or officer (dynamic) — those live in the auth dep on top of this primitive.""" async with self._db() as db: - async with db.execute( + async with await db.execute( _SQL["check_user_has_capability"], (discord_id, capability), ) as cur: @@ -346,7 +341,7 @@ async def role_has_capability( Used by the auth dep to decide whether to bother running the dynamic officer check at all — if officers don't have the capability, no point.""" async with self._db() as db: - async with db.execute( + async with await db.execute( _SQL["check_role_has_capability"], (role, capability), ) as cur: @@ -358,11 +353,11 @@ async def list_role_assignments(self) -> dict[str, list[str]]: Used by the admin user-list endpoint to join roles in without N+1 queries against ``list_roles_for_user``.""" async with self._db() as db: - async with db.execute(_SQL["list_all_role_assignments"]) as cur: + async with await db.execute(_SQL["list_all_role_assignments"]) as cur: rows = await cur.fetchall() out: dict[str, list[str]] = {} - for discord_id, role in rows: - out.setdefault(discord_id, []).append(role) + for row in rows: + out.setdefault(row["discord_id"], []).append(row["role"]) return out diff --git a/backend/server/db/users.sql b/backend/server/db/users.sql index 300d911e..b1268f39 100644 --- a/backend/server/db/users.sql +++ b/backend/server/db/users.sql @@ -1,4 +1,4 @@ --- SQL for backend/server/db/users.py (async aiosqlite). +-- SQL for backend/server/db/users.py (psycopg, users schema). -- Domain: users + user_roles + role_requests + role_permissions. -- --------------------------------------------------------------------------- @@ -7,23 +7,24 @@ -- :name upsert_user INSERT INTO users (discord_id, discord_name, discord_username, avatar, access_status) -VALUES (?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s) ON CONFLICT(discord_id) DO UPDATE SET discord_name = excluded.discord_name, discord_username = excluded.discord_username, avatar = excluded.avatar, - last_seen = strftime('%s','now'), + last_seen = floor(extract(epoch from now())), access_status = CASE - WHEN ? = 1 THEN 'approved' - ELSE access_status + WHEN %s = 1 THEN 'approved' + ELSE users.access_status END; -- :name select_access_status -SELECT access_status FROM users WHERE discord_id = ?; +SELECT access_status FROM users WHERE discord_id = %s; -- :name select_display_names_by_ids --- {placeholders} = "?,?,..." sized at call time -SELECT discord_id, discord_name FROM users WHERE discord_id IN ({placeholders}); +-- The id list binds as ONE array parameter (= ANY) — no composed +-- placeholder strings and no variable-count limits. +SELECT discord_id, discord_name FROM users WHERE discord_id = ANY(%s); -- :name list_pending_users SELECT discord_id, discord_name, discord_username, avatar, first_seen @@ -34,6 +35,8 @@ FROM users WHERE access_status = 'pending' ORDER BY first_seen DESC; UPDATE users SET access_status = 'approved' WHERE access_status = 'pending'; -- :name list_all_users_with_claim_count +-- Bare u.* columns with GROUP BY u.discord_id is legal in Postgres — +-- functional dependency on the users primary key. SELECT u.discord_id, u.discord_name, u.discord_username, u.avatar, u.first_seen, u.last_seen, u.access_status, COUNT(c.id) AS claim_count @@ -43,24 +46,24 @@ GROUP BY u.discord_id ORDER BY u.first_seen DESC; -- :name update_user_access_status -UPDATE users SET access_status = ? WHERE discord_id = ?; +UPDATE users SET access_status = %s WHERE discord_id = %s; -- --------------------------------------------------------------------------- -- user_roles -- --------------------------------------------------------------------------- -- :name grant_role -INSERT INTO user_roles (discord_id, role, granted_by) VALUES (?, ?, ?) +INSERT INTO user_roles (discord_id, role, granted_by) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING; -- :name revoke_role -DELETE FROM user_roles WHERE discord_id = ? AND role = ?; +DELETE FROM user_roles WHERE discord_id = %s AND role = %s; -- :name list_roles_for_user -SELECT role FROM user_roles WHERE discord_id = ? ORDER BY role; +SELECT role FROM user_roles WHERE discord_id = %s ORDER BY role; -- :name check_has_role -SELECT 1 FROM user_roles WHERE discord_id = ? AND role = ? LIMIT 1; +SELECT 1 FROM user_roles WHERE discord_id = %s AND role = %s LIMIT 1; -- :name list_all_role_assignments SELECT discord_id, role FROM user_roles ORDER BY discord_id, role; @@ -70,7 +73,7 @@ SELECT discord_id, role FROM user_roles ORDER BY discord_id, role; -- --------------------------------------------------------------------------- -- :name create_role_request -INSERT INTO role_requests (discord_id, role, user_note) VALUES (?, ?, ?) +INSERT INTO role_requests (discord_id, role, user_note) VALUES (%s, %s, %s) RETURNING id; -- {where_sql} = "WHERE …" or "" composed by Python build_where helper. @@ -92,22 +95,22 @@ SELECT rr.id, rr.discord_id, rr.role, rr.status, u.discord_name, u.discord_username, u.avatar FROM role_requests rr LEFT JOIN users u ON u.discord_id = rr.discord_id -WHERE rr.id = ?; +WHERE rr.id = %s; -- :name review_role_request UPDATE role_requests SET - status = ?, - reviewed_at = strftime('%s','now'), - reviewed_by = ?, - admin_note = ? -WHERE id = ? AND status = 'pending'; + status = %s, + reviewed_at = floor(extract(epoch from now())), + reviewed_by = %s, + admin_note = %s +WHERE id = %s AND status = 'pending'; -- :name select_role_request_grant_info -SELECT discord_id, role FROM role_requests WHERE id = ?; +SELECT discord_id, role FROM role_requests WHERE id = %s; -- :name withdraw_role_request UPDATE role_requests SET status = 'withdrawn' -WHERE id = ? AND discord_id = ? AND status = 'pending'; +WHERE id = %s AND discord_id = %s AND status = 'pending'; -- --------------------------------------------------------------------------- -- role_permissions (capability checks) @@ -117,8 +120,8 @@ WHERE id = ? AND discord_id = ? AND status = 'pending'; SELECT 1 FROM user_roles ur JOIN role_permissions rp ON rp.role = ur.role -WHERE ur.discord_id = ? AND rp.capability = ? +WHERE ur.discord_id = %s AND rp.capability = %s LIMIT 1; -- :name check_role_has_capability -SELECT 1 FROM role_permissions WHERE role = ? AND capability = ? LIMIT 1; +SELECT 1 FROM role_permissions WHERE role = %s AND capability = %s LIMIT 1; diff --git a/backend/server/raid_live.py b/backend/server/raid_live.py index 482efc40..9e503858 100644 --- a/backend/server/raid_live.py +++ b/backend/server/raid_live.py @@ -90,7 +90,7 @@ def team_scheduled_now(team: dict, now: datetime | None = None) -> bool: wd = local.isoweekday() mn = local.hour * 60 + local.minute for r in team.get("raids", []): - days = [int(d) for d in str(r.get("days", "")).split(",") if d] + days = [int(d) for d in (r.get("days") or [])] if days and _slot_active_now(days, r["start_min"], r["end_min"], wd, mn): return True return False From c9061bebf046d8b7f5d87044c6fa6d8965d959c9 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 14:19:07 +0100 Subject: [PATCH 06/32] =?UTF-8?q?feat(pg):=20erasure=20users-half=20on=20p?= =?UTF-8?q?sycopg=20=E2=80=94=20one=20deferred-constraint=20transaction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sqlite users.db connection replaced by pg.connection + search_path; the PRAGMA foreign_keys=OFF ordering dance becomes SET CONSTRAINTS ALL DEFERRED inside a single transaction. attendance_sessions.uploaders is probed with the jsonb ?-operator instead of LIKE and written back via Json(); per-table existence checks dropped (migrations guarantee the schema). Test seam is users_schema= (was users_path=). The parses half stays SQLite until P2 unifies both halves into one transaction. Verified by a live seed-erase-verify smoke on local PG17. Co-Authored-By: Claude Fable 5 --- backend/server/db/erasure.py | 81 ++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/backend/server/db/erasure.py b/backend/server/db/erasure.py index 741a43d8..30447e14 100644 --- a/backend/server/db/erasure.py +++ b/backend/server/db/erasure.py @@ -1,8 +1,9 @@ """Account erasure — the right-to-be-forgotten mechanics behind the privacy policy (2026-09-28). -One sync function over BOTH SQLite files (users.db and parses.db), run from -the routes through ``run_sync``: +One sync function over BOTH stores — the ``users`` Postgres schema and the +parses SQLite file (Postgres in P2, when the halves unify into one +transaction) — run from the routes through ``run_sync``: - rows that ARE the person go: the ``users`` row and everything keyed to their Discord id (tokens, roles, role requests, claims, favourites, @@ -26,13 +27,15 @@ from __future__ import annotations -import json import sqlite3 import time from dataclasses import dataclass, field from pathlib import Path -from backend.server import db as _users_db # DB_PATH read at call time (tests re-point it) +from psycopg.types.json import Json + +from backend import pg +from backend.server import db as _users_db # SCHEMA read at call time (tests re-point it) from backend.server.parses.db import store as parses_store #: The placeholder users row every tombstoned reference points at. @@ -101,11 +104,11 @@ def _table_exists(conn: sqlite3.Connection, table: str) -> bool: return conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone() is not None -def _ensure_tombstone(conn: sqlite3.Connection, now: int) -> None: +def _ensure_tombstone(conn, now: int) -> None: conn.execute( "INSERT INTO users (discord_id, discord_name, discord_username, avatar, first_seen, last_seen, " - "access_status) VALUES (?, 'Deleted user', 'deleted', NULL, ?, ?, 'denied') " - "ON CONFLICT(discord_id) DO NOTHING", + "access_status) VALUES (%s, 'Deleted user', 'deleted', NULL, %s, %s, 'denied') " + "ON CONFLICT (discord_id) DO NOTHING", (DELETED_USER_ID, now, now), ) @@ -113,57 +116,53 @@ def _ensure_tombstone(conn: sqlite3.Connection, now: int) -> None: def erase_user_sync( discord_id: str, *, - users_path: Path | None = None, + users_schema: str | None = None, parses_path: Path | None = None, now: int | None = None, ) -> ErasureResult: - """Erase one Discord account from both databases. Idempotent: a second + """Erase one Discord account from both stores. Idempotent: a second call finds nothing and reports ``found=False``. The tombstone id itself - can never be erased.""" + can never be erased. The users half is ONE Postgres transaction with + constraints deferred (replacing SQLite's PRAGMA foreign_keys=OFF), so a + crash mid-way leaves nothing half-erased; migrations guarantee every + table exists, so the SQLite-era per-table existence probes are gone.""" if not discord_id or discord_id == DELETED_USER_ID: return ErasureResult(found=False) now = int(now if now is not None else time.time()) result = ErasureResult(found=False) - users_db = Path(users_path) if users_path is not None else _users_db.DB_PATH - with sqlite3.connect(users_db) as conn: - conn.execute("PRAGMA foreign_keys = OFF") # we order the deletes ourselves - found_user = conn.execute("SELECT 1 FROM users WHERE discord_id = ?", (discord_id,)).fetchone() + schema = users_schema if users_schema is not None else _users_db.SCHEMA + with pg.connection() as conn: + conn.execute(pg.search_path_sql(schema)) + # The users-referencing FKs are DEFERRABLE INITIALLY IMMEDIATE — + # defer them all so statement order inside this transaction is free. + conn.execute("SET CONSTRAINTS ALL DEFERRED") + found_user = conn.execute("SELECT 1 FROM users WHERE discord_id = %s", (discord_id,)).fetchone() result.found = found_user is not None _ensure_tombstone(conn, now) for table, column in _AUTHOR_COLUMNS: - if not _table_exists(conn, table): - continue - cur = conn.execute(f"UPDATE {table} SET {column} = ? WHERE {column} = ?", (DELETED_USER_ID, discord_id)) + cur = conn.execute(f"UPDATE {table} SET {column} = %s WHERE {column} = %s", (DELETED_USER_ID, discord_id)) if cur.rowcount: result.tombstoned[f"{table}.{column}"] = cur.rowcount for table, column in _OWNED_ROWS: - if not _table_exists(conn, table): - continue - cur = conn.execute(f"DELETE FROM {table} WHERE {column} = ?", (discord_id,)) + cur = conn.execute(f"DELETE FROM {table} WHERE {column} = %s", (discord_id,)) if cur.rowcount: result.deleted[table] = cur.rowcount - if _table_exists(conn, "attendance_observations"): - cur = conn.execute( - "DELETE FROM attendance_observations WHERE kind = 'voice' AND character_name = ?", (discord_id,) - ) - result.voice_observations_deleted = cur.rowcount - if _table_exists(conn, "attendance_sessions"): - rows = conn.execute( - "SELECT id, uploaders FROM attendance_sessions WHERE uploaders LIKE ?", (f"%{discord_id}%",) - ).fetchall() - for session_id, raw in rows: - try: - uploaders = json.loads(raw or "{}") - except ValueError: - continue - if isinstance(uploaders, dict) and discord_id in uploaders: - uploaders.pop(discord_id) - conn.execute( - "UPDATE attendance_sessions SET uploaders = ? WHERE id = ?", (json.dumps(uploaders), session_id) - ) - result.sessions_scrubbed += 1 - cur = conn.execute("DELETE FROM users WHERE discord_id = ?", (discord_id,)) + cur = conn.execute( + "DELETE FROM attendance_observations WHERE kind = 'voice' AND character_name = %s", (discord_id,) + ) + result.voice_observations_deleted = cur.rowcount + # uploaders is jsonb: `?` is the key-exists operator (psycopg only + # treats %s as a placeholder, so the bare ? passes through). + rows = conn.execute( + "SELECT id, uploaders FROM attendance_sessions WHERE uploaders ? %s", (discord_id,) + ).fetchall() + for row in rows: + uploaders = row["uploaders"] or {} + uploaders.pop(discord_id, None) + conn.execute("UPDATE attendance_sessions SET uploaders = %s WHERE id = %s", (Json(uploaders), row["id"])) + result.sessions_scrubbed += 1 + cur = conn.execute("DELETE FROM users WHERE discord_id = %s", (discord_id,)) if cur.rowcount: result.deleted["users"] = cur.rowcount conn.commit() From e608a3c1be6a05bf438dedcbabc7a32807ea040a Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 14:21:12 +0100 Subject: [PATCH 07/32] feat(pg): lifespan owns pools + migrations; drop users init_db from startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit App lifespan now opens the psycopg pools and applies pending migrations (run_sync keeps the runner off the loop; ledger + advisory lock make overlapping deploy containers safe), and closes the pools at shutdown. The bot's defensive users_db.init_db() is gone — pre-lifespan queries fall back to direct connections and guild_context degrades on psycopg.Error. main.py selects the selector event-loop policy on Windows before asyncio.run (psycopg async can't ride ProactorEventLoop). Verified by booting the real lifespan against local PG17: migrations, /api/health, a store call over the pool, clean pool shutdown. Co-Authored-By: Claude Fable 5 --- backend/bot/bot.py | 13 +++++-------- backend/server/app.py | 13 ++++++++++++- main.py | 5 +++++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/backend/bot/bot.py b/backend/bot/bot.py index 944df841..9bd00927 100644 --- a/backend/bot/bot.py +++ b/backend/bot/bot.py @@ -51,14 +51,11 @@ async def setup_hook(self) -> None: from backend.core.logging_config import configure_logging configure_logging() - # The bot shares users.db with the web half but starts concurrently - # with it — the web lifespan owns init_db, so run it defensively - # here too (idempotent CREATE IF NOT EXISTS + guarded migrations) - # before any cog can touch the registry. - from backend.server import db as users_db - - await asyncio.to_thread(users_db.init_db) - + # The bot shares the users schema with the web half and starts + # concurrently with it. The web lifespan owns pools + migrations; + # pre-lifespan bot queries fall back to direct psycopg connections + # (backend/pg.py) and registry lookups degrade on psycopg.Error + # (guild_context), so no defensive init is needed here. self.census = CensusClient(service_id=SERVICE_ID) self.tree.error(self._on_app_command_error) diff --git a/backend/server/app.py b/backend/server/app.py index 094ec3cf..bf7fd376 100644 --- a/backend/server/app.py +++ b/backend/server/app.py @@ -473,7 +473,17 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: f"(web/routes/rankings.py). Set WEB_CONCURRENCY=1 or rewrite " f"both layers to use a cross-process backplane before scaling." ) - users_db.init_db() + # Postgres: open the shared pools, then apply any pending migrations + # — a deploy IS a migration run (ledger + advisory lock make this + # idempotent and safe against overlapping containers). The sync + # runner goes through run_sync so startup never blocks the loop. + from backend import pg, pg_migrate + from backend.server.core.executor import run_sync + + await pg.open_pools() + _applied = await run_sync(pg_migrate.run) + if _applied: + _log.info("[startup] applied %d migration(s): %s", len(_applied), ", ".join(_applied)) # Open-signup backlog clear: when OPEN_SIGNUP is on, auto-approve any # users already waiting in the pending queue so the policy applies to # them too (new logins are auto-approved at upsert time). Idempotent — @@ -554,6 +564,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: # Close the shared aiohttp session(s) so the process exits # without aiohttp's "Unclosed client session" warning. await census_lifecycle.aclose_all() + await pg.close_pools() async def _cache_sweep_loop() -> None: """Periodically evict max_age-expired entries from all caches. diff --git a/main.py b/main.py index 7ebed8f9..35568ab5 100644 --- a/main.py +++ b/main.py @@ -155,4 +155,9 @@ async def main() -> None: if __name__ == "__main__": + # psycopg's async side can't run on Windows' default ProactorEventLoop — + # select the selector policy before the loop exists (no-op on Linux). + from backend import pg + + pg.ensure_selector_event_loop_policy() asyncio.run(main()) From 75a3b9bf04f611d31d8c891b3c86b5e6affe7426 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 14:29:15 +0100 Subject: [PATCH 08/32] feat(pg): PG test infrastructure + facade cleanup; last raw users.db readers converted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/fixtures/pg.py: provision_for_session points pg.dsn() at the local TEST database (never the .env Supabase DSN), creates it if missing, rebuilds session schemas from db/migrations; SchemaLeaser pools scratch schemas (users_s_N) built by retargeting the same 0001 file, reset on release with one TRUNCATE + seeds re-run; users_schema fixture + pg_conn seeding helper. point_users_db_at now takes a schema. Facade: init_db/DB_PATH and the SQLite schema/migrations/_assertions machinery deleted (tests/server/test_db_migrations.py with them). Remaining raw users.db readers converted to the pg seam: characters + guild fallback prefix searches, the character-cache prewarm, the users/claims Prometheus counts (file-size gauge drops users until the P2 pg_total_relation_size split), seed_raid_kills primary lookup. test_guild_settings.py converted as the model file — 11/11 green on PG. Co-Authored-By: Claude Fable 5 --- backend/server/api/character/views.py | 13 +- backend/server/api/characters.py | 10 +- backend/server/api/characters.sql | 2 +- backend/server/api/guild.py | 10 +- backend/server/api/guild.sql | 2 +- backend/server/db/__init__.py | 67 ++------- backend/server/metrics.py | 28 ++-- backend/server/metrics.sql | 4 +- scripts/dev/seed_raid_kills.py | 28 ++-- tests/conftest.py | 43 +++--- tests/fixtures/pg.py | 190 ++++++++++++++++++++++++++ tests/fixtures/users_db.py | 35 ++--- tests/server/test_guild_settings.py | 16 +-- 13 files changed, 299 insertions(+), 149 deletions(-) create mode 100644 tests/fixtures/pg.py diff --git a/backend/server/api/character/views.py b/backend/server/api/character/views.py index 0dcac4e7..3e8f28e3 100644 --- a/backend/server/api/character/views.py +++ b/backend/server/api/character/views.py @@ -358,15 +358,14 @@ def _build_char_response(char) -> CharacterResponse: async def _prewarm_for_world(world: str, sem: asyncio.Semaphore) -> None: """Pre-warm the character cache for all approved claims in one world.""" - import aiosqlite - - from backend.server.db import DB_PATH + from backend import pg + from backend.server.db import SCHEMA as _USERS_SCHEMA try: - async with aiosqlite.connect(DB_PATH) as db: - db.row_factory = aiosqlite.Row - async with db.execute( - "SELECT DISTINCT character_name FROM character_claims WHERE status = 'approved' AND world = ?", + async with pg.aconnection() as db: + await db.execute(pg.search_path_sql(_USERS_SCHEMA)) + async with await db.execute( + "SELECT DISTINCT character_name FROM character_claims WHERE status = 'approved' AND world = %s", (world,), ) as cur: names = [row["character_name"] for row in await cur.fetchall()] diff --git a/backend/server/api/characters.py b/backend/server/api/characters.py index fa1f3d3d..7e0f4a18 100644 --- a/backend/server/api/characters.py +++ b/backend/server/api/characters.py @@ -2,17 +2,17 @@ import logging -import aiosqlite from fastapi import APIRouter, Request from pydantic import BaseModel +from backend import pg from backend.census.store import store as census_store from backend.core.log_safety import scrub from backend.server.cache import character_cache from backend.server.core.cache_keys import char_cache_key from backend.server.core.census_lifecycle import shared_census_client from backend.server.core.executor import run_sync -from backend.server.db import DB_PATH +from backend.server.db import SCHEMA as _USERS_SCHEMA from backend.server.limiter import limiter from backend.server.server_context import current_world from backend.sql_loader import load_sql @@ -65,9 +65,9 @@ def _store_search_sync(q: str, world: str) -> list[CharNameResult]: async def _local_search(q: str) -> list[CharNameResult]: - async with aiosqlite.connect(DB_PATH) as db: - db.row_factory = aiosqlite.Row - async with db.execute(_SQL["local_search_by_prefix"], (f"{q.lower()}%",)) as cur: + async with pg.aconnection() as db: + await db.execute(pg.search_path_sql(_USERS_SCHEMA)) + async with await db.execute(_SQL["local_search_by_prefix"], (f"{q.lower()}%",)) as cur: rows = await cur.fetchall() return [CharNameResult(name=r["character_name"]) for r in rows] diff --git a/backend/server/api/characters.sql b/backend/server/api/characters.sql index 89b56649..829a281f 100644 --- a/backend/server/api/characters.sql +++ b/backend/server/api/characters.sql @@ -5,7 +5,7 @@ -- distinct names ordered alphabetically. Fallback when Census is unavailable. SELECT DISTINCT character_name FROM character_claims -WHERE LOWER(character_name) LIKE ? +WHERE LOWER(character_name) LIKE %s AND status IN ('approved', 'pending') ORDER BY character_name LIMIT 50; diff --git a/backend/server/api/guild.py b/backend/server/api/guild.py index c32dd214..c4166e01 100644 --- a/backend/server/api/guild.py +++ b/backend/server/api/guild.py @@ -4,10 +4,10 @@ import logging import time -import aiosqlite from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel +from backend import pg from backend.census.store import store as census_store from backend.core.log_safety import scrub as _scrub from backend.server.cache import guild_cache @@ -22,7 +22,7 @@ from backend.server.core.census_lifecycle import shared_census_client from backend.server.core.executor import run_sync from backend.server.core.validation import validate_guild_name as _validate_guild_name_lib -from backend.server.db import DB_PATH as _USERS_DB_PATH +from backend.server.db import SCHEMA as _USERS_SCHEMA from backend.server.db import get_active_claims from backend.server.guild_cache import ( _bg_refresh_guild, @@ -529,9 +529,9 @@ def _store_guild_search() -> list[str]: return GuildSearchResponse(results=[GuildNameResult(name=n) for n in store_names], total=len(store_names)) # Census failed — fall back to locally-tracked guilds in item_watch - async with aiosqlite.connect(_USERS_DB_PATH) as db: - db.row_factory = aiosqlite.Row - async with db.execute(_SQL["local_guild_search_by_prefix"], (f"{q.lower()}%",)) as cur: + async with pg.aconnection() as db: + await db.execute(pg.search_path_sql(_USERS_SCHEMA)) + async with await db.execute(_SQL["local_guild_search_by_prefix"], (f"{q.lower()}%",)) as cur: rows = await cur.fetchall() results = [GuildNameResult(name=r["guild_name"]) for r in rows] diff --git a/backend/server/api/guild.sql b/backend/server/api/guild.sql index b6e808bd..71cd76d0 100644 --- a/backend/server/api/guild.sql +++ b/backend/server/api/guild.sql @@ -6,6 +6,6 @@ -- unavailable. SELECT DISTINCT guild_name FROM item_watch -WHERE LOWER(guild_name) LIKE ? +WHERE LOWER(guild_name) LIKE %s ORDER BY guild_name LIMIT 25; diff --git a/backend/server/db/__init__.py b/backend/server/db/__init__.py index b030ba44..ad0aa505 100644 --- a/backend/server/db/__init__.py +++ b/backend/server/db/__init__.py @@ -1,68 +1,29 @@ -"""Backend user/claims/tokens/servers DB layer. +"""Backend user/claims/tokens/servers DB layer — the users Postgres schema. -Carved out of the original 1309-line web/db.py. Five unrelated domains -each get their own module: +Carved out of the original 1309-line web/db.py. Each domain gets its own +module (users, claims, item_watch, tokens, servers, …); the schema DDL +lives in db/migrations/0001_users.sql, applied by backend/pg_migrate.py +from the app lifespan (deploy = migrate) and by the test fixtures. - - users.py — users table + role/role_request/role_permission helpers - - claims.py — character_claims table - - item_watch.py — item_watch table - - tokens.py — api_tokens table - - servers.py — servers (per-server registry) table - -The init_db() orchestrator + the DB_PATH constant live here. Every -per-domain helper is re-exported from this module so the existing -`from web import db as users_db; users_db.get_active_claims(...)` API -shape is preserved — no consumer rewrites needed. +Every per-domain helper is re-exported from this module so the existing +`from backend.server import db as users_db; users_db.get_active_claims(...)` +API shape is preserved — no consumer rewrites needed. """ from __future__ import annotations -import sqlite3 -from pathlib import Path - -from backend.db_helpers import resolve_db_path -from backend.server.db._assertions import assert_schema_complete -from backend.server.db.migrations import apply_migrations -from backend.server.db.schema import SCHEMA as _SQLITE_SCHEMA - -DB_PATH = resolve_db_path("DB_USERS_PATH", "users.db") - #: Postgres schema the users family lives in (see db/migrations/ #: 0001_users.sql). Tests re-point each store's ``schema`` at leased -#: scratch schemas; DB_PATH above survives only until the remaining -#: SQLite consumers (erasure's parses half) finish migrating. +#: scratch schemas via tests/fixtures/users_db.point_users_db_at. SCHEMA = "users" -def init_db(path: Path | None = None) -> None: - """Create tables if they don't exist + apply migrations. - - Called once at startup. Idempotent. Order: - 1. executescript SCHEMA — creates tables + the indices known at v1. - 2. apply_migrations(conn) — ALTER TABLE + post-ALTER index creates + seeds. - - Memory [[test-migrations-against-old-db-shape]]: any new column added - here MUST be added to BOTH SCHEMA (for fresh DBs) and migrations.py - (for existing DBs). Column-dependent indexes live in migrations.py - AFTER the ADD COLUMN — never in SCHEMA. - """ - # Read DB_PATH at call time, not def time — the default-arg capture - # pattern is exactly what the store conversion eliminated everywhere - # else, and conftest re-points DB_PATH after import (BE-096 race). - path = Path(path) if path is not None else DB_PATH - path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(path) as conn: - conn.executescript(_SQLITE_SCHEMA) - apply_migrations(conn) - assert_schema_complete(conn) - - # --------------------------------------------------------------------------- # Facade: re-export each domain store's bound methods so the existing # `users_db.get_active_claims(...)` API shape is preserved. The domains are -# XStore(AsyncStoreBase) classes now (backend/db_catalogue.py) — the bound -# methods read the shared instance's `path` dynamically, so conftest -# re-points one attribute per store and every alias follows. +# XStore(PgStoreBase) classes (backend/db_catalogue.py) — the bound methods +# read the shared instance's `schema` dynamically, so conftest re-points one +# attribute per store and every alias follows. # --------------------------------------------------------------------------- from backend.server.db.aa_plans import store as aa_plans_store # noqa: E402 @@ -138,8 +99,8 @@ def init_db(path: Path | None = None) -> None: user_has_capability_via_db = users_store.user_has_capability_via_db withdraw_role_request = users_store.withdraw_role_request -#: Every domain store over users.db — conftest re-points `store.path` on -#: each after re-resolving DB_PATH from the env. +#: Every domain store over the users schema — tests re-point +#: `store.schema` on each (tests/fixtures/users_db.point_users_db_at). ALL_STORES = ( aa_plans_store, attendance_store, diff --git a/backend/server/metrics.py b/backend/server/metrics.py index 15338263..1f48fd95 100644 --- a/backend/server/metrics.py +++ b/backend/server/metrics.py @@ -249,8 +249,9 @@ def _close_conn(self, name: str) -> None: def collect(self): # type: ignore[override] # Lazy imports — keep metrics.py importable from tests without the # full DB modules loaded. + from backend import pg from backend.eq2db import raids as raids_db - from backend.server.db import DB_PATH as users_db_path + from backend.server.db import SCHEMA as users_schema from backend.server.parses import db as parses_db g_users = GaugeMetricFamily("users_total", "Registered users by access status", labels=["status"]) @@ -273,20 +274,21 @@ def collect(self): # type: ignore[override] "ACT spell-timer definitions stored across all encounters", ) - # users.db ---------------------------------------------------------- - conn = self._get_conn("users", users_db_path) - if conn is not None: - try: + # users schema (Postgres) -------------------------------------------- + # Tiny COUNTs per scrape; a short-lived pooled checkout beats keeping + # a scrape-lifetime connection around. (The P2 metrics split replaces + # this with 60s-cached counts.) + try: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(users_schema)) for status in ("approved", "pending", "denied"): row = conn.execute(_SQL["count_users_by_access_status"], (status,)).fetchone() - g_users.add_metric([status], row[0] if row else 0) - + g_users.add_metric([status], row["n"] if row else 0) for status in ("pending", "approved", "rejected", "withdrawn", "superseded"): row = conn.execute(_SQL["count_claims_by_status"], (status,)).fetchone() - g_claims.add_metric([status], row[0] if row else 0) - except Exception: - _log.exception("[metrics] users.db collector error") - self._close_conn("users") + g_claims.add_metric([status], row["n"] if row else 0) + except Exception: + _log.exception("[metrics] users-schema collector error") # parses.db — encounters split by hidden_at (visible vs soft-deleted) # so dashboards can distinguish "live leaderboard rows" from @@ -342,12 +344,12 @@ def collect(self): # type: ignore[override] from backend.eq2db import recipes as recipes_db from backend.eq2db import spells as spells_db from backend.eq2db import zones as zones_db - from backend.server.db import DB_PATH as users_db_path from backend.server.parses import db as parses_db # Map label → Path. Centralised so adding a new DB is one tuple. + # (users moved to Postgres — its size gauge returns in the P2 + # metrics split as pg_total_relation_size per schema.) candidates = [ - ("users", users_db_path), ("parses", parses_db.DB_PATH), ("census", census_store.DB_PATH), ("raids", raids_db.DB_PATH), diff --git a/backend/server/metrics.sql b/backend/server/metrics.sql index 4130d164..f746e68b 100644 --- a/backend/server/metrics.sql +++ b/backend/server/metrics.sql @@ -1,10 +1,10 @@ -- SQL for backend/server/metrics.py — Prometheus collector COUNT queries. -- :name count_users_by_access_status -SELECT COUNT(*) FROM users WHERE access_status = ?; +SELECT COUNT(*) AS n FROM users WHERE access_status = %s; -- :name count_claims_by_status -SELECT COUNT(*) FROM character_claims WHERE status = ?; +SELECT COUNT(*) AS n FROM character_claims WHERE status = %s; -- :name count_visible_encounters SELECT COUNT(*) FROM encounters WHERE hidden_at IS NULL; diff --git a/scripts/dev/seed_raid_kills.py b/scripts/dev/seed_raid_kills.py index 9329b0e6..1ca91c63 100644 --- a/scripts/dev/seed_raid_kills.py +++ b/scripts/dev/seed_raid_kills.py @@ -33,9 +33,10 @@ from dotenv import load_dotenv # noqa: E402 +from backend import pg # noqa: E402 from backend.census.client import CensusClient # noqa: E402 from backend.eq2db import zones as zones_db # noqa: E402 -from backend.server.db import DB_PATH as USERS_DB_PATH # noqa: E402 +from backend.server.db import SCHEMA as USERS_SCHEMA # noqa: E402 from backend.server.parses.db import DB_PATH as PARSES_DB_PATH # noqa: E402 load_dotenv(_REPO / ".env") @@ -78,19 +79,22 @@ def _read_primary_character() -> tuple[str, str] | None: """Return ``(discord_id, character_name)`` of the user's primary claim. Picks the oldest approved+primary claim if multiple exist (single-user dev - DBs effectively only have one). Returns None if no primary is set. + DBs effectively only have one). Returns None if no primary is set (or the + users schema is unreachable/unmigrated). """ - if not USERS_DB_PATH.exists(): + try: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(USERS_SCHEMA)) + row = conn.execute( + """ + SELECT discord_id, character_name FROM character_claims + WHERE status = 'approved' AND is_primary = 1 + ORDER BY requested_at ASC LIMIT 1 + """, + ).fetchone() + except Exception: return None - with sqlite3.connect(USERS_DB_PATH) as conn: - row = conn.execute( - """ - SELECT discord_id, character_name FROM character_claims - WHERE status = 'approved' AND is_primary = 1 - ORDER BY requested_at ASC LIMIT 1 - """, - ).fetchone() - return (row[0], row[1]) if row else None + return (row["discord_id"], row["character_name"]) if row else None async def _resolve_guild_via_census(character_name: str, world: str, attempts: int = 4) -> str | None: diff --git a/tests/conftest.py b/tests/conftest.py index 2da2869d..bb8b1367 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,14 +2,16 @@ Test isolation note ------------------- -Both ``web.db.DB_PATH`` and ``parses.db.DB_PATH`` are evaluated at module -import time. To stop the test suite from touching the developer's real -``data/users.db`` / ``data/parses/parses.db`` (the production files), we -redirect both via env vars **before** any ``web.*`` import below. +SQLite families: ``parses.db.DB_PATH`` (and the catalogue paths) are +evaluated at module import time. To stop the test suite from touching the +developer's real data files, we redirect them via env vars **before** any +``web.*`` import below. The tmp dir is wiped at the start of every pytest +session, so tests start from an empty DB every run. -The tmp dir is wiped at the start of every pytest session, so tests start -from an empty DB every run. Per-test isolation is then up to individual -fixtures / mocks — most tests already mock the DB-touching calls outright. +Postgres (users family): tests/fixtures/pg.py points ``pg.dsn()`` at the +local TEST database (TEST_DATABASE_URL) and rebuilds the session schemas +from db/migrations/ — never the developer's .env Supabase DSN. Per-test +isolation comes from the ``users_schema`` fixture (leased scratch schemas). BE-096: env vars are set inside ``pytest_configure`` (a plugin-ordered hook that runs after plugin discovery, before test collection) to avoid a race @@ -63,7 +65,6 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 BE-096: moved from module-level os.environ calls to avoid a race with pytest plugins (e.g. pytest-asyncio) that may import web.app during plugin discovery.""" - os.environ["DB_USERS_PATH"] = str(_TEST_DB_DIR / "users.db") os.environ["DB_PARSES_PATH"] = str(_TEST_DB_DIR / "backend.server.parses.db") os.environ["DB_CENSUS_PATH"] = str(_TEST_DB_DIR / "backend.census.db") os.environ["DB_ZONES_PATH"] = str(_TEST_DB_DIR / "zones.db") @@ -89,6 +90,18 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 # with HTTPS_ONLY=true in their env still gets a working test run. os.environ["HTTPS_ONLY"] = "false" + # --- Postgres (users family) ------------------------------------- + # psycopg's async side can't run on Windows' ProactorEventLoop; the + # policy must be set before pytest-asyncio creates any loop. + from backend import pg as _pg + + _pg.ensure_selector_event_loop_policy() + # Point pg.dsn() at the local TEST database (never the .env Supabase + # DSN) and rebuild the session schemas from the migration files. + from tests.fixtures.pg import provision_for_session + + provision_for_session() + # Imports below this line read the env vars above when they evaluate their # module-level constants (DB_PATH, SESSION_SECRET, ...). from backend.census import store as census_store @@ -108,14 +121,10 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 from backend.eq2db import recipes as recipes_db from backend.eq2db import spells as spells_db from backend.eq2db import zones as zones_db - from backend.server import db as users_db from backend.server.parses import db as parses_db parses_db.DB_PATH = resolve_db_path("DB_PARSES_PATH", "parses", "parses.db") parses_db.store.path = parses_db.DB_PATH - users_db.DB_PATH = resolve_db_path("DB_USERS_PATH", "users.db") - for _store in users_db.ALL_STORES: - _store.path = users_db.DB_PATH census_store.DB_PATH = resolve_db_path("DB_CENSUS_PATH", "census", "census.db") census_store.store.path = census_store.DB_PATH # eq2db catalogue modules: re-point both the module constant AND the @@ -131,11 +140,10 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 mod.DB_PATH = resolve_db_path(env_var, subdir, filename) mod.catalogue.path = mod.DB_PATH - # Create both schemas immediately. FastAPI's startup hooks (which would - # normally call init_db) don't fire under ASGITransport, so without this - # step API-token / parses tests would hit a missing-table OperationalError - # the first time they read from the DB. - users_db.init_db() + # Create the parses schema immediately. FastAPI's startup hooks don't + # fire under ASGITransport, so without this step parses tests would hit + # a missing-table OperationalError the first time they read from the DB. + # (The users family is provisioned by provision_for_session above.) parses_db.store.init_db() @@ -226,6 +234,7 @@ def mock_character_cache(): # directory (the fixtures' module location is implementation detail). from tests.fixtures.logging_state import _logging_state_isolation # noqa: F401,E402 from tests.fixtures.parses_db import parses_db_conn, parses_db_path # noqa: F401,E402 +from tests.fixtures.pg import users_schema # noqa: F401,E402 @pytest.fixture(autouse=True) diff --git a/tests/fixtures/pg.py b/tests/fixtures/pg.py new file mode 100644 index 00000000..2217507a --- /dev/null +++ b/tests/fixtures/pg.py @@ -0,0 +1,190 @@ +"""Shared Postgres test plumbing for the migrated users family. + +Two isolation tiers, mirroring the old SQLite arrangement: + +- SESSION schema: ``pytest_configure`` (conftest) calls + :func:`provision_for_session`, which points ``pg.dsn()`` at the local + TEST database (never the developer's .env Supabase DSN), rebuilds the + production-named schemas from the migration files, and leaves them + shared for the whole run — the analog of the old shared tmpdir + users.db that route tests ran against. +- SCRATCH schemas: the :data:`leaser` hands out ``users_s_N`` + schemas built by retargeting the SAME migration file (header + substitution — one source of truth), reset on release with one + ``TRUNCATE … RESTART IDENTITY CASCADE`` + a re-run of the idempotent + ``-- seeds`` section. The :func:`users_schema` fixture leases one and + points every store at it — the analog of ``init_db(tmp_path)`` + + ``point_users_db_at``. + +:func:`pg_conn` is the seeding/assertion replacement for the old raw +``sqlite3.connect(users_db)`` blocks. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import psycopg +import pytest +from psycopg import conninfo +from psycopg.rows import dict_row + +from backend import pg, pg_migrate + +#: Local-only default (the winget PG17 unattended install); override with +#: TEST_DATABASE_URL in the environment or .env. +_DEFAULT_TEST_DSN = "postgresql://postgres:postgres@localhost:5432/eq2lexicon_test" + +_INSTALL_HINT = ( + "The test suite needs a local PostgreSQL 17 server:\n" + " winget install PostgreSQL.PostgreSQL.17\n" + "then make sure TEST_DATABASE_URL (default: localhost:5432, database\n" + "eq2lexicon_test, user/password postgres) is reachable." +) + + +def resolve_test_dsn() -> str: + return os.environ.get("TEST_DATABASE_URL") or _DEFAULT_TEST_DSN + + +def provision_for_session() -> None: + """Point the pg seam at the TEST database and rebuild the session + schemas from the migration files. Called once from pytest_configure + (xdist workers skip the destructive rebuild — the controller already + did it). Fails collection with an install hint if PG is unreachable.""" + test_dsn = resolve_test_dsn() + # DATABASE_URL is the highest-precedence name pg.dsn() reads, so this + # guarantees no test can ever write to the .env Supabase DSN. + os.environ["DATABASE_URL"] = test_dsn + + params: dict[str, Any] = conninfo.conninfo_to_dict(test_dsn) + dbname = str(params.get("dbname") or "eq2lexicon_test") + try: + try: + psycopg.connect(test_dsn, connect_timeout=3).close() + except psycopg.OperationalError as exc: + if "does not exist" not in str(exc): + raise + # The server is up but the test database is missing — create it + # via the maintenance DB (CREATE DATABASE can't run in a txn). + admin = conninfo.make_conninfo(**{**params, "dbname": "postgres"}) + with psycopg.connect(admin, autocommit=True, connect_timeout=3) as conn: + conn.execute(psycopg.sql.SQL("CREATE DATABASE {}").format(psycopg.sql.Identifier(dbname))) + except psycopg.OperationalError as exc: + pytest.exit( + f"Cannot reach the test PostgreSQL database {dbname!r}: {exc}\n{_INSTALL_HINT}", + returncode=4, + ) + + if os.environ.get("PYTEST_XDIST_WORKER"): + return + + with pg.connection() as conn: + # Drop the ledger so pg_migrate re-applies everything, then the + # session schemas and any scratch schemas a crashed run left over. + conn.execute("DROP TABLE IF EXISTS public.schema_migrations") + conn.execute("DROP SCHEMA IF EXISTS users CASCADE") + rows = conn.execute("SELECT nspname FROM pg_namespace WHERE nspname LIKE 'users\\_s%'").fetchall() + for r in rows: + conn.execute(f'DROP SCHEMA IF EXISTS "{r["nspname"]}" CASCADE') + conn.commit() + pg_migrate.run() + + +@contextmanager +def pg_conn(schema: str = "users") -> Iterator[Any]: + """Dict-row sync connection with ``search_path`` set — the test + seeding / assertion replacement for ``sqlite3.connect(users_db)``. + Commits on clean exit; ``%s`` params, rows are dicts.""" + with pg.connection() as conn: + conn.execute(pg.search_path_sql(schema)) + yield conn + conn.commit() + + +class SchemaLeaser: + """Lazily-grown pool of scratch users-family schemas. + + Fresh schemas are built by retargeting the same migration file the + production schema uses, so scratch and session schemas can never + drift. ``release()`` resets with one TRUNCATE + seeds re-run + (~10-20ms) instead of a 100-250ms CREATE SCHEMA + full DDL per test.""" + + def __init__(self, family: str = "users", migration: str = "0001_users.sql") -> None: + self.family = family + self._migration = pg_migrate.MIGRATIONS_DIR / migration + self._free: list[str] = [] + self._count = 0 + self._tables: list[str] | None = None + self._seeds: str | None = None + + def _sql_text(self) -> str: + return self._migration.read_text(encoding="utf-8") + + def _seeds_sql(self) -> str: + if self._seeds is None: + text = self._sql_text() + marker = text.find("-- seeds") + if marker < 0: + raise AssertionError(f"{self._migration.name} has no '-- seeds' marker") + self._seeds = text[marker:] + return self._seeds + + def _create(self) -> str: + self._count += 1 + # pid suffix: parallel xdist workers share one test database. + name = f"{self.family}_s{os.getpid()}_{self._count}" + with pg.connection() as conn: + conn.execute(f'DROP SCHEMA IF EXISTS "{name}" CASCADE') + conn.execute(pg_migrate.retarget(self._sql_text(), name)) + conn.commit() + return name + + def acquire(self) -> str: + return self._free.pop() if self._free else self._create() + + def release(self, name: str) -> None: + try: + self._reset(name) + except Exception: + return # never pool a schema we failed to clean + self._free.append(name) + + def _reset(self, name: str) -> None: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(name)) + if self._tables is None: + rows = conn.execute("SELECT tablename FROM pg_tables WHERE schemaname = %s", (name,)).fetchall() + self._tables = sorted(r["tablename"] for r in rows) + tables = ", ".join(f'"{t}"' for t in self._tables) + conn.execute(f"TRUNCATE {tables} RESTART IDENTITY CASCADE") + conn.execute(self._seeds_sql()) + conn.commit() + + +#: The session-wide leaser every test shares. +leaser = SchemaLeaser() + + +@pytest.fixture +def users_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: + """A leased scratch users-family schema with every store pointed at + it — the per-test-isolated replacement for ``init_db(tmp_path / + 'users.db')`` + ``point_users_db_at(monkeypatch, path)``.""" + from tests.fixtures.users_db import point_users_db_at + + name = leaser.acquire() + point_users_db_at(monkeypatch, name) + try: + yield name + finally: + leaser.release(name) + + +def connect_test_db() -> Any: + """A bare dict-row connection to the test database (no search_path) — + for the rare test that inspects cross-schema state.""" + return psycopg.connect(resolve_test_dsn(), row_factory=dict_row) # type: ignore[arg-type] diff --git a/tests/fixtures/users_db.py b/tests/fixtures/users_db.py index 86b00dda..78addc6b 100644 --- a/tests/fixtures/users_db.py +++ b/tests/fixtures/users_db.py @@ -1,34 +1,27 @@ -"""Shared users.db test plumbing. +"""Shared users-family test plumbing. -The seven users.db domain stores each carry their own ``path`` (captured -from DB_PATH at import). Re-pointing users.db for a test therefore means -re-pointing the module constant AND every store instance — this helper is -the one place that knows that, so per-file loops can't fork (some copies -had drifted to skip the DB_PATH half). +Every users-family store carries a ``schema`` attribute (default +``"users"``, the session schema). Re-pointing the family for a test +means re-pointing the facade constant AND every store instance — this +helper is the one place that knows that, so per-file loops can't fork. -Usage in a fixture:: +Most tests should just take the ``users_schema`` fixture +(tests/fixtures/pg.py), which leases a clean scratch schema and calls +this for you:: - from tests.fixtures.users_db import point_users_db_at - - @pytest.fixture(autouse=True) - def _users_tmp(tmp_path, monkeypatch): - db_file = tmp_path / "users.db" - init_db(db_file) - point_users_db_at(monkeypatch, db_file) - return db_file + def test_something(users_schema): + ... # every store now points at an isolated schema """ from __future__ import annotations -from pathlib import Path - import pytest from backend.server import db as users_db -def point_users_db_at(monkeypatch: pytest.MonkeyPatch, path: Path) -> None: - """Re-point the users.db module constant + every domain store at ``path``.""" - monkeypatch.setattr(users_db, "DB_PATH", path) +def point_users_db_at(monkeypatch: pytest.MonkeyPatch, schema: str) -> None: + """Re-point the facade constant + every domain store at ``schema``.""" + monkeypatch.setattr(users_db, "SCHEMA", schema) for store in users_db.ALL_STORES: - monkeypatch.setattr(store, "path", path) + monkeypatch.setattr(store, "schema", schema) diff --git a/tests/server/test_guild_settings.py b/tests/server/test_guild_settings.py index 0a5bf502..140d89d9 100644 --- a/tests/server/test_guild_settings.py +++ b/tests/server/test_guild_settings.py @@ -9,30 +9,22 @@ import base64 import json -from pathlib import Path from unittest.mock import AsyncMock, patch import itsdangerous import pytest from httpx import ASGITransport, AsyncClient -from backend.server.db import init_db from backend.server.db.guild_settings import store as gs -from tests.fixtures.users_db import point_users_db_at _TEST_SECRET = "pytest-session-secret-not-real-0123456789" -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema # --------------------------------------------------------------------------- From 1e68f77705dd8a7ecaea7cca02662e26490b53b8 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 14:32:49 +0100 Subject: [PATCH 09/32] ci: postgres:17 service for the python job Users-family tests now run against real PostgreSQL; tests/fixtures/pg.py reads TEST_DATABASE_URL and provisions schemas from db/migrations/. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64c7eeb2..f3f79940 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,24 @@ jobs: python: name: Python (lint + types + tests) runs-on: ubuntu-latest + services: + # Users-family tests run against real PostgreSQL (tests/fixtures/pg.py + # provisions schemas from db/migrations/); SQLite families still use + # tmpdir files. Match the local dev install's major version. + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: eq2lexicon_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/eq2lexicon_test steps: - uses: actions/checkout@v7 From 5c895c12146ed0039f42a217028a97a59353734f Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 14:34:25 +0100 Subject: [PATCH 10/32] =?UTF-8?q?test(pg):=20meta-tests=20=E2=80=94=20SQL?= =?UTF-8?q?=20block=20references,=20PG=20comment=20hazard,=20migration=20r?= =?UTF-8?q?unner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/test_sql_blocks.py pins every _SQL[...] reference to an existing sidecar block (the dropped-block class) and rejects format-field comments that survive parsing inside PG-dialect block bodies (psycopg counts %s inside comments). tests/test_pg_migrate.py enforces the two-line family header convention retarget() depends on, and exercises the ledger: idempotent re-runs, per-file rollback leaving failures unrecorded. Co-Authored-By: Claude Fable 5 --- tests/test_pg_migrate.py | 99 ++++++++++++++++++++++++++++++++++++++++ tests/test_sql_blocks.py | 76 ++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 tests/test_pg_migrate.py create mode 100644 tests/test_sql_blocks.py diff --git a/tests/test_pg_migrate.py b/tests/test_pg_migrate.py new file mode 100644 index 00000000..42412018 --- /dev/null +++ b/tests/test_pg_migrate.py @@ -0,0 +1,99 @@ +"""pg_migrate: the family-header convention meta-test + runner behaviour. + +The two-line header convention is load-bearing: the test scratch-schema +leaser retargets migration files by substituting exactly those lines, so +a file that drifts from the convention silently breaks per-test schema +isolation. The runner tests exercise the real ledger against the test +database (conftest provisions it). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from backend import pg, pg_migrate + + +def test_every_family_migration_follows_the_two_line_header() -> None: + for path in pg_migrate.migration_files(): + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + assert lines, f"{path.name} is empty" + if lines[0].strip() == "-- global": + with pytest.raises(ValueError, match="global"): + pg_migrate.retarget(text, "scratch_x") + continue + # retarget() itself enforces the convention — it must not raise, + # and the rewritten header must land on the requested schema. + rewritten = pg_migrate.retarget(text, "scratch_x") + head = rewritten.splitlines()[:2] + assert head[0] == "create schema if not exists scratch_x;" + assert head[1] == "set search_path to scratch_x, public;" + + +def test_retarget_rejects_nonconforming_files() -> None: + with pytest.raises(ValueError, match="two-line family header"): + pg_migrate.retarget("CREATE TABLE x (id int);", "scratch_x") + + +def test_runner_is_idempotent_and_records_the_ledger(tmp_path: Path) -> None: + """A scratch migration directory applied twice: first run applies and + records, second run is a no-op. Uses its own schema + ledger rows so + the session schemas are untouched.""" + (tmp_path / "9001_scratch_family.sql").write_text( + "create schema if not exists pgmig_scratch;\n" + "set search_path to pgmig_scratch, public;\n" + "CREATE TABLE widgets (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL);\n", + encoding="utf-8", + ) + try: + first = pg_migrate.run(directory=tmp_path) + assert first == ["9001_scratch_family.sql"] + second = pg_migrate.run(directory=tmp_path) + assert second == [] + with pg.connection() as conn: + row = conn.execute( + "SELECT 1 AS ok FROM public.schema_migrations WHERE filename = %s", + ("9001_scratch_family.sql",), + ).fetchone() + assert row is not None + conn.execute("INSERT INTO pgmig_scratch.widgets (name) VALUES ('w')") + got = conn.execute("SELECT id, name FROM pgmig_scratch.widgets").fetchone() + assert got == {"id": 1, "name": "w"} + conn.commit() + finally: + with pg.connection() as conn: + conn.execute("DROP SCHEMA IF EXISTS pgmig_scratch CASCADE") + conn.execute( + "DELETE FROM public.schema_migrations WHERE filename = %s", + ("9001_scratch_family.sql",), + ) + conn.commit() + + +def test_failed_migration_rolls_back_and_is_not_recorded(tmp_path: Path) -> None: + (tmp_path / "9002_bad_family.sql").write_text( + "create schema if not exists pgmig_bad;\n" + "set search_path to pgmig_bad, public;\n" + "CREATE TABLE ok_table (id int);\n" + "CREATE TABLE broken (id int REFERENCES does_not_exist(id));\n", + encoding="utf-8", + ) + try: + with pytest.raises(Exception): # noqa: B017 — any psycopg error is fine + pg_migrate.run(directory=tmp_path) + with pg.connection() as conn: + row = conn.execute( + "SELECT 1 AS ok FROM public.schema_migrations WHERE filename = %s", + ("9002_bad_family.sql",), + ).fetchone() + assert row is None, "failed migration must not be recorded" + # the whole file rolled back — ok_table must not exist either + reg = conn.execute("SELECT to_regclass('pgmig_bad.ok_table') AS r").fetchone() + assert reg is not None and reg["r"] is None + finally: + with pg.connection() as conn: + conn.execute("DROP SCHEMA IF EXISTS pgmig_bad CASCADE") + conn.commit() diff --git a/tests/test_sql_blocks.py b/tests/test_sql_blocks.py new file mode 100644 index 00000000..3102864a --- /dev/null +++ b/tests/test_sql_blocks.py @@ -0,0 +1,76 @@ +"""Meta-test: every ``_SQL["name"]`` reference resolves in its .sql sidecar. + +A store conversion once dropped a named block while editing the comment +above it (claims.sql, select_claim_user_and_world) — the failure only +surfaced when that exact code path ran. This pins the whole repo: any +reference to a missing block fails collection-fast, dialect-free. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from backend.sql_loader import parse_sql + +_REPO = Path(__file__).resolve().parents[1] +_REF_RE = re.compile(r'_SQL\["([a-z0-9_]+)"\]') + + +def _module_sql_pairs() -> list[tuple[Path, Path]]: + pairs = [] + for sql_path in sorted((_REPO / "backend").rglob("*.sql")): + py_path = sql_path.with_suffix(".py") + if py_path.is_file(): + pairs.append((py_path, sql_path)) + return pairs + + +@pytest.mark.parametrize( + ("py_path", "sql_path"), + _module_sql_pairs(), + ids=lambda p: str(p.relative_to(_REPO)) if isinstance(p, Path) else str(p), +) +def test_every_sql_reference_resolves(py_path: Path, sql_path: Path) -> None: + refs = set(_REF_RE.findall(py_path.read_text(encoding="utf-8"))) + blocks = set(parse_sql(sql_path.read_text(encoding="utf-8"))) + missing = refs - blocks + assert not missing, f"{py_path.name} references missing SQL blocks: {sorted(missing)}" + + +#: SQLite-dialect sidecars (families not yet on Postgres) — sqlite parses +#: comments server-side, so a format field inside a comment is harmless +#: there. Shrink this set as P2/P3 convert each family. +_SQLITE_FAMILIES = { + "backend/eq2db", + "backend/census", + "backend/server/parses", + "backend/server/api", # parses/zones route sidecars still ? dialect + "backend/image", + "backend/bot", +} + +_PG_DIRS = ("backend/server/db",) + + +def test_no_format_fields_inside_pg_block_bodies() -> None: + """A ``-- … {field}`` comment that SURVIVES block parsing (i.e. is not + a trailing comment the loader trims) gets str.format()ed with the body; + the substituted %s inside the comment then counts as a psycopg + placeholder (psycopg does not strip comments client-side) and breaks + the parameter count — claims.sql list_claims shipped exactly this. + Scoped to the Postgres-dialect sidecars.""" + offenders: list[str] = [] + for _, sql_path in _module_sql_pairs(): + rel = sql_path.relative_to(_REPO).as_posix() + if not rel.startswith(_PG_DIRS): + continue + blocks = parse_sql(sql_path.read_text(encoding="utf-8")) + for name, body in blocks.items(): + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("--") and re.search(r"\{[a-z_]+\}", stripped): + offenders.append(f"{sql_path.name} block {name!r}: {stripped!r}") + assert not offenders, f"format-field comments inside parsed PG block bodies: {offenders}" From ceb095172feceabc80ecbd29d9cb0cc63925740d Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:03:45 +0100 Subject: [PATCH 11/32] chore(pg): remove retired SQLite users schema/migrations machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deletions staged for the test-infrastructure commit were silently unstaged by an intervening stash cycle — landing them now. Co-Authored-By: Claude Fable 5 --- backend/server/db/_assertions.py | 76 ----- backend/server/db/migrations.py | 150 --------- backend/server/db/migrations.sql | 190 ------------ backend/server/db/schema.py | 18 -- backend/server/db/schema.sql | 483 ----------------------------- tests/server/test_db_migrations.py | 319 ------------------- 6 files changed, 1236 deletions(-) delete mode 100644 backend/server/db/_assertions.py delete mode 100644 backend/server/db/migrations.py delete mode 100644 backend/server/db/migrations.sql delete mode 100644 backend/server/db/schema.py delete mode 100644 backend/server/db/schema.sql delete mode 100644 tests/server/test_db_migrations.py diff --git a/backend/server/db/_assertions.py b/backend/server/db/_assertions.py deleted file mode 100644 index 1440d8b2..00000000 --- a/backend/server/db/_assertions.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Schema-vs-migrations drift assertion. Run from init_db AFTER migrations, -so any column added to SCHEMA but missed in migrations.py raises loudly at -test startup rather than crashing at runtime on a pre-migration-shape DB. - -Memory [[test-migrations-against-old-db-shape]]: this is the codified form -of the lesson learnt when a column-dependent index in SCHEMA crashed the -production startup against an existing DB. -""" - -from __future__ import annotations - -import re -import sqlite3 - -from backend.server.db.schema import SCHEMA - - -def _split_column_defs(columns_block: str) -> list[str]: - """Split a CREATE TABLE column list on top-level commas only. - - DEFAULT (strftime('%s','now')) contains a comma inside parentheses; a - naive str.split(",") would break that into two fragments and misidentify - the second fragment as a column name. This function counts paren depth - and only splits when depth == 0. - """ - parts: list[str] = [] - depth = 0 - current: list[str] = [] - for ch in columns_block: - if ch == "(": - depth += 1 - current.append(ch) - elif ch == ")": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - parts.append("".join(current)) - current = [] - else: - current.append(ch) - if current: - parts.append("".join(current)) - return parts - - -def assert_schema_complete(conn: sqlite3.Connection) -> None: - """Raise AssertionError if any column in SCHEMA is missing from the live DB. - - Parses SCHEMA for every ``CREATE TABLE IF NOT EXISTS`` block, extracts - the expected column names, then cross-checks each against - ``PRAGMA table_info``. Any column that appears in SCHEMA but is absent - from the live DB means a migration is missing. - - Only runs when assertions are enabled (the default for pytest / dev); - in production with ``python -O`` the assertion body is compiled out and - this is a no-op — so prod is unaffected by this check. - """ - # Parse SCHEMA for every "CREATE TABLE IF NOT EXISTS x (col1 ..., col2 ...)" block. - create_re = re.compile(r"CREATE TABLE IF NOT EXISTS (\w+) \(([^;]+)\);", re.DOTALL) - for table_name, columns_block in create_re.findall(SCHEMA): - expected_cols = { - line.strip().split()[0] - for line in _split_column_defs(columns_block) - if line.strip() - and not line.strip().upper().startswith("PRIMARY") - and not line.strip().upper().startswith("FOREIGN") - and not line.strip().startswith("--") - and not line.strip().startswith("UNIQUE") - } - actual_cols = {row[1] for row in conn.execute(f"PRAGMA table_info({table_name})")} - missing = expected_cols - actual_cols - assert not missing, ( - f"Schema drift on table {table_name!r}: SCHEMA declares " - f"columns {missing} that don't exist on the live DB. The migration " - f"for these columns is missing from web/db/migrations.py." - ) diff --git a/backend/server/db/migrations.py b/backend/server/db/migrations.py deleted file mode 100644 index 320e0008..00000000 --- a/backend/server/db/migrations.py +++ /dev/null @@ -1,150 +0,0 @@ -"""ALTER TABLE migrations for users.db. - -Each section adds a column or rebuilds a table for a constraint change. -Idempotent — guarded on PRAGMA table_info checks so re-runs are no-ops. - -Pattern: - cols = {row[1] for row in conn.execute("PRAGMA table_info(X)")} - if "new_col" not in cols: - conn.execute(_SQL["alter_X_add_new_col"]) - conn.execute(_SQL["index_X_new"]) - -SQL statements live in migrations.sql alongside. The Python guards stay -here because they're not SQL — they're "WHEN to apply the ALTER" logic. - -Memory [[test-migrations-against-old-db-shape]]: every consumer-visible -column-add MUST also be reflected in schema.sql for the fresh-DB path. -Drift between the two has bitten the repo before (a column-dependent -index in SCHEMA crashed prod startup against an existing DB). -""" - -from __future__ import annotations - -import os -import sqlite3 - -from backend.sql_loader import load_sql - -_SQL = load_sql(__file__) - - -def apply_migrations(conn: sqlite3.Connection) -> None: - """Apply every idempotent ALTER + post-ALTER index + seed. Called from init_db.""" - # Migrate: add columns introduced after initial schema - users_cols = {row[1] for row in conn.execute("PRAGMA table_info(users)")} - if "discord_username" not in users_cols: - conn.execute(_SQL["alter_users_add_discord_username"]) - if "access_status" not in users_cols: - conn.execute(_SQL["alter_users_add_access_status"]) - conn.execute(_SQL["backfill_users_access_status_approved"]) - - conn.execute(_SQL["index_users_access"]) - - claims_cols = {row[1] for row in conn.execute("PRAGMA table_info(character_claims)")} - if "requested_at" not in claims_cols: - conn.execute(_SQL["alter_claims_add_requested_at"]) - conn.execute(_SQL["backfill_claims_requested_at"]) - if "reviewed_at" not in claims_cols: - conn.execute(_SQL["alter_claims_add_reviewed_at"]) - if "reviewed_by" not in claims_cols: - conn.execute(_SQL["alter_claims_add_reviewed_by"]) - if "note" not in claims_cols: - conn.execute(_SQL["alter_claims_add_note"]) - if "is_primary" not in claims_cols: - conn.execute(_SQL["alter_claims_add_is_primary"]) - if "world" not in claims_cols: - conn.execute(_SQL["alter_claims_add_world"]) - conn.execute(_SQL["index_claims_world"]) - conn.execute(_SQL["index_claims_primary"]) - - # Migrate item_watch: add world column + rebuild to change UNIQUE constraint. - watch_cols = {row[1] for row in conn.execute("PRAGMA table_info(item_watch)")} - if "world" not in watch_cols: - conn.executescript(_SQL["rebuild_item_watch_add_world"]) - - # Drop the officer→edit_content permission seeded by earlier versions. - conn.execute(_SQL["drop_officer_edit_content"]) - - conn.executemany( - _SQL["seed_role_permissions"], - [ - ("contributor", "edit_content"), - ], - ) - - # Seed the servers registry (idempotent). Varsoon takes the current env - # values; Wuoshi starts with defaults edited later in admin. - from backend.census import config as _cfg - - conn.execute( - _SQL["seed_server"], - ( - "Varsoon", - "varsoon", - "Varsoon", - _cfg.SERVER_MAX_LEVEL, - os.getenv("SERVER_CURRENT_XPAC") or None, - _cfg.LAUNCH_DT_ISO or None, - ), - ) - conn.execute( - _SQL["seed_server"], - ("Wuoshi", "wuoshi", "Wuoshi", _cfg.SERVER_MAX_LEVEL, os.getenv("SERVER_CURRENT_XPAC") or None, None), - ) - - # Migrate: add is_default column (absent from the pre-existing prod servers - # table). MUST run after the seed INSERTs above so it is safe to UPDATE on - # first boot. Nothing is_default-dependent may live in schema.sql. - servers_cols = {row[1] for row in conn.execute("PRAGMA table_info(servers)")} - if "is_default" not in servers_cols: - conn.execute(_SQL["alter_servers_add_is_default"]) - # Migrate: upcoming-expansion countdown fields (2026-09). Same rule as - # is_default — the pre-existing prod servers table lacks them. - if "next_xpac" not in servers_cols: - conn.execute(_SQL["alter_servers_add_next_xpac"]) - if "next_xpac_dt" not in servers_cols: - conn.execute(_SQL["alter_servers_add_next_xpac_dt"]) - # Migrate: xpac-rollover stamp (2026-09) — the rankings era-lock cutoff. - if "current_xpac_started_dt" not in servers_cols: - conn.execute(_SQL["alter_servers_add_current_xpac_started_dt"]) - # Ensure exactly one default exists. On a fresh DB or after ADD COLUMN every - # row has is_default=0 until we set one. Default to the EQ2_WORLD server. - if conn.execute(_SQL["count_default_servers"]).fetchone()[0] == 0: - conn.execute(_SQL["set_server_default_by_world"], (_cfg.WORLD,)) - # If EQ2_WORLD isn't a known server, fall back to the first row alphabetically. - if conn.execute(_SQL["count_default_servers"]).fetchone()[0] == 0: - conn.execute(_SQL["set_server_default_fallback"]) - - # Migrate: placeholder-raider fields on raid_roster_roles (2026-09) — - # census-hidden characters added by hand carry placeholder=1 + a class. - roles_cols = {row[1] for row in conn.execute("PRAGMA table_info(raid_roster_roles)")} - if "placeholder" not in roles_cols: - conn.execute(_SQL["alter_roles_add_placeholder"]) - if "cls" not in roles_cols: - conn.execute(_SQL["alter_roles_add_cls"]) - - # Migrate: parse-posting fields on discord_guild_links (2026-09) — the - # bot posts new raid parses to this channel; the watermark stops reposts. - links_cols = {row[1] for row in conn.execute("PRAGMA table_info(discord_guild_links)")} - if "parses_channel_id" not in links_cols: - conn.execute(_SQL["alter_links_add_parses_channel"]) - if "parses_posted_at" not in links_cols: - conn.execute(_SQL["alter_links_add_parses_posted_at"]) - - # Migrate: newest-wins availability (2026-09) — user self-declarations - # gain an edit stamp so officer character entries can override stale - # ones. Legacy rows default 0 (any stamped edit beats them). - avail_cols = {row[1] for row in conn.execute("PRAGMA table_info(user_availability)")} - if "updated_at" not in avail_cols: - conn.execute(_SQL["alter_user_availability_add_updated_at"]) - - # Migrate: guild_recruitment re-keyed by census guild id (2026-10). - # guild_id joins the PK, which ALTER TABLE can't do, so this is a - # rebuild. The name-keyed first cut existed only on dev boxes for a few - # hours pre-release (never deployed) and its rows carry no census id to - # map onto the NOT NULL PK — the rebuild starts empty. - recruit_cols = {row[1] for row in conn.execute("PRAGMA table_info(guild_recruitment)")} - if recruit_cols and "guild_id" not in recruit_cols: - conn.executescript(_SQL["rebuild_guild_recruitment_guild_id"]) - - conn.commit() diff --git a/backend/server/db/migrations.sql b/backend/server/db/migrations.sql deleted file mode 100644 index 60bbd7fe..00000000 --- a/backend/server/db/migrations.sql +++ /dev/null @@ -1,190 +0,0 @@ --- ALTER TABLE migrations for users.db. --- --- Each statement here is idempotent at the SQL level — the Python guards in --- migrations.py (PRAGMA table_info checks) decide WHETHER to run them based --- on the column state. CREATE INDEX IF NOT EXISTS / INSERT OR IGNORE blocks --- are safe to run every boot regardless of guard. - --- --------------------------------------------------------------------------- --- users --- --------------------------------------------------------------------------- - --- :name alter_users_add_discord_username -ALTER TABLE users ADD COLUMN discord_username TEXT; - --- :name alter_users_add_access_status -ALTER TABLE users ADD COLUMN access_status TEXT NOT NULL DEFAULT 'pending'; - --- Existing users were already using the app — approve them all. Only --- brand-new users (after this migration) start as 'pending'. --- :name backfill_users_access_status_approved -UPDATE users SET access_status = 'approved'; - --- BE-073: notifications endpoint polls list_pending_users every 60s for every --- logged-in admin — without this index, WHERE access_status = 'pending' is a --- full table scan. Selectivity is high (most users are approved), so the index --- pays for itself. --- :name index_users_access -CREATE INDEX IF NOT EXISTS idx_users_access ON users(access_status); - --- --------------------------------------------------------------------------- --- character_claims --- --------------------------------------------------------------------------- - --- SQLite ALTER TABLE does not support non-constant defaults, so we add the --- column as nullable and backfill existing rows with the current epoch. --- :name alter_claims_add_requested_at -ALTER TABLE character_claims ADD COLUMN requested_at INTEGER; - --- :name backfill_claims_requested_at -UPDATE character_claims SET requested_at = strftime('%s','now') WHERE requested_at IS NULL; - --- :name alter_claims_add_reviewed_at -ALTER TABLE character_claims ADD COLUMN reviewed_at INTEGER; - --- :name alter_claims_add_reviewed_by -ALTER TABLE character_claims ADD COLUMN reviewed_by TEXT; - --- :name alter_claims_add_note -ALTER TABLE character_claims ADD COLUMN note TEXT; - --- :name alter_claims_add_is_primary -ALTER TABLE character_claims ADD COLUMN is_primary INTEGER NOT NULL DEFAULT 0; - --- :name alter_claims_add_world -ALTER TABLE character_claims ADD COLUMN world TEXT NOT NULL DEFAULT 'Varsoon'; - --- Index on world is created here (not in schema.sql) so it runs AFTER the --- ADD COLUMN above. IF NOT EXISTS keeps subsequent boots no-op. --- :name index_claims_world -CREATE INDEX IF NOT EXISTS idx_claims_world ON character_claims(world); - --- BE-072: supports the "primary claim per (user, world)" filter that hits --- get_active_claims on every read. --- :name index_claims_primary -CREATE INDEX IF NOT EXISTS idx_claims_primary ON character_claims(discord_id, world, is_primary); - --- --------------------------------------------------------------------------- --- item_watch (rebuild to change UNIQUE constraint — SQLite can't ALTER it) --- --------------------------------------------------------------------------- - --- Multi-statement rebuild; run via conn.executescript. Guarded on the --- absence of the `world` column so it fires exactly once on a pre-existing --- DB; no-op on fresh DBs (which get the new shape directly via schema.sql). --- :name rebuild_item_watch_add_world -ALTER TABLE item_watch RENAME TO item_watch_old; -CREATE TABLE item_watch ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL DEFAULT 'Varsoon', - guild_name TEXT NOT NULL, - character_name TEXT NOT NULL, - item_id INTEGER NOT NULL, - item_name TEXT NOT NULL, - added_by TEXT NOT NULL REFERENCES users(discord_id), - added_by_name TEXT NOT NULL, - added_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - first_seen_at INTEGER, - last_seen_at INTEGER, - last_checked_at INTEGER, - UNIQUE(world, guild_name, character_name, item_id) -); -INSERT INTO item_watch (id, world, guild_name, character_name, item_id, item_name, - added_by, added_by_name, added_at, first_seen_at, last_seen_at, last_checked_at) - SELECT id, 'Varsoon', guild_name, character_name, item_id, item_name, - added_by, added_by_name, added_at, first_seen_at, last_seen_at, last_checked_at - FROM item_watch_old; -DROP TABLE item_watch_old; -CREATE INDEX IF NOT EXISTS idx_watch_guild ON item_watch(guild_name); - --- --------------------------------------------------------------------------- --- role_permissions — seed + drift cleanup --- --------------------------------------------------------------------------- - --- Drop the officer→edit_content permission seeded by earlier versions. --- Officer access to raid editing was removed (2026-05-29 decision); only --- admins + contributors edit content now. No-op once the row is gone. --- :name drop_officer_edit_content -DELETE FROM role_permissions WHERE role = 'officer' AND capability = 'edit_content'; - --- Idempotent seeds — listed here (not schema.sql) so seeds live with the --- migration sequence that introduced them. --- :name seed_role_permissions -INSERT OR IGNORE INTO role_permissions (role, capability) VALUES (?, ?); - --- --------------------------------------------------------------------------- --- servers — registry seed + is_default migration --- --------------------------------------------------------------------------- - --- :name seed_server -INSERT OR IGNORE INTO servers (world, subdomain, display_name, max_level, current_xpac, launch_dt) -VALUES (?, ?, ?, ?, ?, ?); - --- :name alter_servers_add_is_default -ALTER TABLE servers ADD COLUMN is_default INTEGER NOT NULL DEFAULT 0; - --- :name alter_servers_add_next_xpac -ALTER TABLE servers ADD COLUMN next_xpac TEXT; - --- :name alter_servers_add_next_xpac_dt -ALTER TABLE servers ADD COLUMN next_xpac_dt TEXT; - --- :name alter_servers_add_current_xpac_started_dt -ALTER TABLE servers ADD COLUMN current_xpac_started_dt TEXT; - --- :name count_default_servers -SELECT COUNT(*) FROM servers WHERE is_default = 1; - --- :name set_server_default_by_world -UPDATE servers SET is_default = 1 WHERE world = ?; - --- Fall back to the first row alphabetically when EQ2_WORLD isn't a known server. --- :name set_server_default_fallback -UPDATE servers SET is_default = 1 WHERE world = -(SELECT world FROM servers ORDER BY display_name LIMIT 1); - --- :name alter_roles_add_placeholder -ALTER TABLE raid_roster_roles ADD COLUMN placeholder INTEGER NOT NULL DEFAULT 0; - --- :name alter_roles_add_cls -ALTER TABLE raid_roster_roles ADD COLUMN cls TEXT; - --- :name alter_links_add_parses_channel -ALTER TABLE discord_guild_links ADD COLUMN parses_channel_id TEXT; - --- :name alter_links_add_parses_posted_at -ALTER TABLE discord_guild_links ADD COLUMN parses_posted_at INTEGER NOT NULL DEFAULT 0; - --- :name alter_user_availability_add_updated_at -ALTER TABLE user_availability ADD COLUMN updated_at INTEGER NOT NULL DEFAULT 0; - --- --------------------------------------------------------------------------- --- guild_recruitment — re-key from (world, guild_name) to the census guild id --- --------------------------------------------------------------------------- - --- Multi-statement rebuild; run via conn.executescript. Guarded on the --- absence of the `guild_id` column. The name-keyed first cut of this table --- existed only on dev boxes for a few hours pre-release (never deployed) --- and its rows carry no census id to map onto the NOT NULL PK -- so the --- rebuild starts empty. Keep the column list in step with schema.sql. --- :name rebuild_guild_recruitment_guild_id -DROP TABLE guild_recruitment; -CREATE TABLE guild_recruitment ( - world TEXT NOT NULL, - guild_id INTEGER NOT NULL, - guild_name TEXT NOT NULL, - recruiting INTEGER NOT NULL DEFAULT 0, - description TEXT NOT NULL DEFAULT '', - classes_json TEXT NOT NULL DEFAULT '[]', - tags_json TEXT NOT NULL DEFAULT '[]', - contacts_json TEXT NOT NULL DEFAULT '[]', - discord_url TEXT, - updated_by TEXT, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - logo BLOB, - logo_media_type TEXT, - logo_uploaded_by TEXT, - logo_uploaded_at INTEGER, - PRIMARY KEY (world, guild_id) -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_guild_recruitment_name ON guild_recruitment(world, guild_name); -CREATE INDEX IF NOT EXISTS idx_guild_recruitment_listing ON guild_recruitment(world, recruiting); diff --git a/backend/server/db/schema.py b/backend/server/db/schema.py deleted file mode 100644 index 00c04a08..00000000 --- a/backend/server/db/schema.py +++ /dev/null @@ -1,18 +0,0 @@ -"""DDL — current shape of every table. - -The actual CREATE TABLE / CREATE INDEX text lives in schema.sql alongside. -This module just loads it and exposes a single ``SCHEMA`` string so the -two existing consumers (init_db and the _assertions test) keep working. - -ONLY safe-to-rerun statements belong in schema.sql. Any column-dependent -statement (e.g. CREATE INDEX on a column added by ALTER) MUST live in -migrations.py AFTER the corresponding ADD COLUMN — putting it in schema.sql -would silently crash on existing DBs (see memory -[[test-migrations-against-old-db-shape]]). -""" - -from __future__ import annotations - -from backend.sql_loader import load_sql - -SCHEMA: str = load_sql(__file__)["all"] diff --git a/backend/server/db/schema.sql b/backend/server/db/schema.sql deleted file mode 100644 index c986e9a5..00000000 --- a/backend/server/db/schema.sql +++ /dev/null @@ -1,483 +0,0 @@ --- Current DDL shape of every table owned by backend/server/db. --- --- ONLY contains CREATE TABLE / CREATE INDEX statements that are safe to run --- on both a fresh DB and an existing DB. Any column-dependent statement --- (CREATE INDEX on a column added by ALTER) MUST live in migrations.sql, --- run AFTER the corresponding ADD COLUMN. Adding such a statement here will --- silently crash on existing DBs. --- --- The single `all` block is run via conn.executescript() in __init__.init_db. --- A test (_assertions.py) reads the same block and asserts schema-level --- invariants. - --- :name all -PRAGMA journal_mode = WAL; -PRAGMA foreign_keys = ON; - -CREATE TABLE IF NOT EXISTS users ( - discord_id TEXT PRIMARY KEY, - discord_name TEXT NOT NULL, - discord_username TEXT, - avatar TEXT, - first_seen INTEGER NOT NULL DEFAULT (strftime('%s','now')), - last_seen INTEGER NOT NULL DEFAULT (strftime('%s','now')), - access_status TEXT NOT NULL DEFAULT 'pending' -); - -CREATE TABLE IF NOT EXISTS character_claims ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - discord_id TEXT NOT NULL REFERENCES users(discord_id), - character_name TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - requested_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - reviewed_at INTEGER, - reviewed_by TEXT, - note TEXT, - world TEXT NOT NULL DEFAULT 'Varsoon' -); - -CREATE INDEX IF NOT EXISTS idx_claims_discord ON character_claims(discord_id); -CREATE INDEX IF NOT EXISTS idx_claims_status ON character_claims(status); --- NOTE: the index on character_claims(world) is NOT created here. The schema --- runs via executescript BEFORE the ALTER that adds `world` to a pre-existing --- table, so creating it here would raise "no such column: world" on an --- existing DB. It is created in migrations.sql after the ADD COLUMN --- migration instead. - -CREATE TABLE IF NOT EXISTS item_watch ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL DEFAULT 'Varsoon', - guild_name TEXT NOT NULL, - character_name TEXT NOT NULL, - item_id INTEGER NOT NULL, - item_name TEXT NOT NULL, - added_by TEXT NOT NULL REFERENCES users(discord_id), - added_by_name TEXT NOT NULL, - added_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - first_seen_at INTEGER, -- first time we saw them wearing it (NULL = never) - last_seen_at INTEGER, -- most recent check where they had it equipped - last_checked_at INTEGER, -- most recent check (any result) - UNIQUE(world, guild_name, character_name, item_id) -); - -CREATE INDEX IF NOT EXISTS idx_watch_guild ON item_watch(guild_name); - --- Persistent, admin-grantable roles. --- --- The role-source layering for content-edit gates is: --- admin — env-driven (ADMIN_DISCORD_IDS). Stays out of this table so --- a DB wipe can't lock admins out. --- contributor — DB-driven via this table (admin-grantable from the UI). --- officer — dynamic, computed from Census guild rank at request time; --- never persisted here. --- --- One row per (user, role) pair. Adding a new role to the system is data, not --- schema — just start inserting rows under a new role name and gate it where --- appropriate. --- --- TODO(future): per-role permission system. Today the role → capability --- mapping is hardcoded inside `require_editor` (and any future require_X). --- When the codebase grows >1 capability dimensions, layer a `role_permissions` --- table (role TEXT, capability TEXT) on top of this and have the deps consult --- it instead of hardcoded role names. Until then YAGNI. -CREATE TABLE IF NOT EXISTS user_roles ( - discord_id TEXT NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE, - role TEXT NOT NULL, - granted_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - granted_by TEXT NOT NULL, -- discord_id of the granting admin - PRIMARY KEY (discord_id, role) -); - -CREATE INDEX IF NOT EXISTS idx_user_roles_role ON user_roles(role); - --- Per-role capability map. The route-layer auth gate (`require_capability` --- in web/auth_deps.py) JOINs user_roles ↔ role_permissions on `role` to --- answer "does this user have capability X?". --- --- Admin is the synthetic "all capabilities" branch and never appears here. --- Officer DOES appear here even though it's not stored in user_roles — the --- dep dynamically resolves officer status when it sees an ('officer', X) --- row and the user lacks the capability via DB roles. That keeps adding a --- new capability for officer a one-row INSERT rather than a code change. --- --- New capability = INSERT rows here (admins/contributors/officers as --- appropriate). Re-seeding is idempotent via INSERT OR IGNORE in init_db. -CREATE TABLE IF NOT EXISTS role_permissions ( - role TEXT NOT NULL, - capability TEXT NOT NULL, - PRIMARY KEY (role, capability) -); - -CREATE INDEX IF NOT EXISTS idx_role_permissions_capability - ON role_permissions(capability); - --- Self-service role requests. Mirrors the character_claims queue pattern: --- users submit, admin reviews, request transitions through statuses. On --- approval the route also writes a row into user_roles so the request + --- the grant are decoupled (an approved request is immutable history; the --- grant can be independently revoked). --- --- Status transitions: --- pending — submitted by the user, awaiting admin review --- approved — admin approved + user_roles row inserted --- rejected — admin rejected (admin_note may carry the reason) --- withdrawn — user cancelled their own pending request -CREATE TABLE IF NOT EXISTS role_requests ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - discord_id TEXT NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE, - role TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - requested_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - reviewed_at INTEGER, - reviewed_by TEXT, -- admin's discord_id - user_note TEXT, -- "why I want this" note - admin_note TEXT -- admin's response note -); - -CREATE INDEX IF NOT EXISTS idx_role_requests_status ON role_requests(status); -CREATE INDEX IF NOT EXISTS idx_role_requests_discord ON role_requests(discord_id); - --- Only one pending request per (user, role) — a second submit while one's --- in flight is rejected by the route. Resolved requests (approved/rejected/ --- withdrawn) can coexist for the same (user, role) for the audit trail. -CREATE UNIQUE INDEX IF NOT EXISTS idx_role_requests_one_pending - ON role_requests(discord_id, role) WHERE status = 'pending'; - -CREATE TABLE IF NOT EXISTS api_tokens ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id TEXT NOT NULL REFERENCES users(discord_id), - name TEXT NOT NULL, -- user-given label e.g. "Desktop ACT" - token_hash TEXT NOT NULL UNIQUE, -- sha256 hex of the raw token - token_prefix TEXT NOT NULL, -- first 12 chars for UI display (eq2c_ + 7 chars) - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - last_used_at INTEGER, -- updated on each successful auth - revoked_at INTEGER -- non-NULL = inactive -); - -CREATE INDEX IF NOT EXISTS idx_tokens_user ON api_tokens(user_id); -CREATE INDEX IF NOT EXISTS idx_tokens_hash ON api_tokens(token_hash); - -CREATE TABLE IF NOT EXISTS servers ( - world TEXT PRIMARY KEY, - subdomain TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - max_level INTEGER NOT NULL, - current_xpac TEXT, - launch_dt TEXT, - next_xpac TEXT, - next_xpac_dt TEXT, - -- Stamped by the automatic xpac rollover (backend/server/xpac_rollover.py) - -- with the instant current_xpac went live. The rankings era-lock cutoff: - -- out-of-era raid parses ingested after this stop entering leaderboards. - current_xpac_started_dt TEXT, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - is_default INTEGER NOT NULL DEFAULT 0 -); --- NOTE: no index or statement referencing `is_default` here. SCHEMA runs via --- executescript BEFORE the ADD COLUMN migration on a pre-existing DB, so any --- column-dependent DDL/DML must live in migrations.sql after the ALTER, never --- here. - --- Guild raid schedules. Officer-editable, publicly viewable. Up to 4 teams per --- guild (team_index 0..3); each team has up to 4 raids (raid_slots, slot_index --- 0..3). Brand-new tables — CREATE TABLE IF NOT EXISTS is safe on existing DBs, --- so no migration is needed (unlike an ADD COLUMN). --- NOTE: inline comments in these column blocks must NOT contain commas — --- _assertions.py splits column defs on top-level commas to check schema drift. -CREATE TABLE IF NOT EXISTS raid_teams ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL, - guild_name TEXT NOT NULL, - team_index INTEGER NOT NULL, -- 0..3 (display order) - name TEXT NOT NULL, -- e.g. Team 1 (officer-editable) - primary_tz TEXT NOT NULL, -- IANA tz e.g. America/New_York - twitch_login TEXT, -- normalized channel login (nullable) - updated_by TEXT NOT NULL REFERENCES users(discord_id), - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - UNIQUE(world, guild_name, team_index) -); - -CREATE INDEX IF NOT EXISTS idx_raid_teams_guild ON raid_teams(world, guild_name); - -CREATE TABLE IF NOT EXISTS raid_slots ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - team_id INTEGER NOT NULL REFERENCES raid_teams(id) ON DELETE CASCADE, - slot_index INTEGER NOT NULL, -- 0..3 - days TEXT NOT NULL, -- CSV of ISO weekdays (1=Mon .. 7=Sun) - start_min INTEGER NOT NULL, -- minutes since midnight in the team tz (0..1439) - end_min INTEGER NOT NULL, -- may be < start_min so crosses midnight; span <= 300 (5h) - label TEXT, -- optional (e.g. Progression) - UNIQUE(team_id, slot_index) -); - -CREATE INDEX IF NOT EXISTS idx_raid_slots_team ON raid_slots(team_id); - --- Favourite (bookmark) characters per user. NOT ownership — carries no guild --- or claim implications. Scoped per (character, world) so each server's home --- page shows only that world's favourites. Brand-new table — CREATE TABLE IF --- NOT EXISTS is safe on existing DBs so no migration is needed. --- Names are validated + capitalised at the route layer before any DB access --- so the plain UNIQUE behaves case-insensitively. -CREATE TABLE IF NOT EXISTS character_favorites ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - discord_id TEXT NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE, - character_name TEXT NOT NULL, -- canonical capitalised EQ2 name - world TEXT NOT NULL, - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - UNIQUE(discord_id, character_name, world) -); - --- Serves the public favourited-by-N count on character pages. The per-user --- list query is covered by the UNIQUE index prefix (discord_id first). -CREATE INDEX IF NOT EXISTS idx_favorites_character ON character_favorites(character_name, world); - --- ── Download counters ──────────────────────────────────────────────────────── --- One row per (user, download slug) so the public count on the Downloads page --- is distinct-downloaders and can't be inflated by one person re-clicking. --- Slugs are a fixed allowlist in the API (parser-setup / parser-portable / --- act-plugin). Not world-scoped — the parser + plugin are universal. -CREATE TABLE IF NOT EXISTS download_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - discord_id TEXT NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE, - slug TEXT NOT NULL, -- download identifier eg parser-setup - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - UNIQUE(discord_id, slug) -); - --- Serves the per-slug COUNT(*) on the Downloads page. -CREATE INDEX IF NOT EXISTS idx_download_events_slug ON download_events(slug); - --- ── Raid planning ──────────────────────────────────────────────────────────── --- Officer-curated raid rosters + per-team group layouts + per-user availability. --- All three are brand-new tables so CREATE TABLE IF NOT EXISTS needs no --- migration. Names are stored as they appear in the Census roster; lookups --- lower-case both sides at the query layer. - --- Which guild characters are on the raid roster and in what capacity. --- Guild-scoped (not team-scoped): a character is a raider for the guild and --- may then be placed into any team's layout. -CREATE TABLE IF NOT EXISTS raid_roster_roles ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL, - guild_name TEXT NOT NULL, - character_name TEXT NOT NULL, - role TEXT NOT NULL, -- raider or raid_alt - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_by TEXT, -- discord id of the officer - placeholder INTEGER NOT NULL DEFAULT 0, -- 1 = census-hidden character added by hand - cls TEXT, -- class label for placeholder rows - UNIQUE(world, guild_name, character_name) -); - -CREATE INDEX IF NOT EXISTS idx_raid_roles_guild ON raid_roster_roles(world, guild_name); - --- ── Raid attendance ───────────────────────────────────────────────────────── --- Canonical merged attendance per guild raid night. Multiple officers upload --- cumulative parser snapshots; ingest folds them into one session via --- time-gap clustering. The scheduled flag is frozen at ingest so later --- schedule edits never rewrite history. -CREATE TABLE IF NOT EXISTS attendance_sessions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL, - guild_name TEXT NOT NULL, -- canonical Census casing - session_day TEXT NOT NULL, -- ISO date bucket (6h evening rollover) - seq INTEGER NOT NULL DEFAULT 0, -- disambiguates double-headers - started_at INTEGER NOT NULL, -- min first_seen across merged snapshots - ended_at INTEGER NOT NULL, -- max last_seen across merged snapshots - zones TEXT NOT NULL DEFAULT '[]', -- JSON array of raid zones seen - scheduled INTEGER NOT NULL DEFAULT 0, -- 1 = overlapped a raid-schedule window - team_index INTEGER, -- matched raid_teams index - uploaders TEXT NOT NULL DEFAULT '{}', -- JSON audit of contributing discord ids - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - UNIQUE(world, guild_name, session_day, seq) -); - -CREATE INDEX IF NOT EXISTS idx_att_sessions_guild - ON attendance_sessions(world, guild_name, started_at); - --- One merged row per (session <-> character <-> kind). kind raid = in the --- raid; kind online = guildmate online during the window; kind voice is --- reserved for the Phase 3 bot (name column carries a discord id there). --- The min/max upsert is commutative so uploader arrival order is irrelevant. -CREATE TABLE IF NOT EXISTS attendance_observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id INTEGER NOT NULL REFERENCES attendance_sessions(id) ON DELETE CASCADE, - character_name TEXT NOT NULL, -- canonical capitalised EQ2 name - kind TEXT NOT NULL, -- raid or online (voice in phase 3) - first_seen INTEGER NOT NULL, - last_seen INTEGER NOT NULL, - UNIQUE(session_id, character_name, kind) -); - --- Where each rostered character sits in a team's 4x6 layout. Keyed by --- team_index (position in the guild's raid_teams list) rather than team id: --- replace_schedule regenerates team rows so ids are not stable across --- schedule edits. Placements for team indexes beyond the current team count --- are pruned when the schedule is saved. -CREATE TABLE IF NOT EXISTS raid_placements ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL, - guild_name TEXT NOT NULL, - team_index INTEGER NOT NULL, -- 0..3 - character_name TEXT NOT NULL, - group_num INTEGER, -- 1..4 and NULL when benched or sat out - slot INTEGER, -- 0..5 within the group - sitout INTEGER NOT NULL DEFAULT 0, -- 1 = parked on the sitout strip - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_by TEXT, - UNIQUE(world, guild_name, team_index, character_name) -); - -CREATE INDEX IF NOT EXISTS idx_raid_place_team ON raid_placements(world, guild_name, team_index); - --- Per-user raid availability calendar. Only non-default days are stored — --- an absent row means Available. Global per user (not per guild or world): --- a player is AFK on a date regardless of which character they bring. -CREATE TABLE IF NOT EXISTS user_availability ( - discord_id TEXT NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE, - day TEXT NOT NULL, -- ISO date YYYY-MM-DD - status TEXT NOT NULL, -- tentative or afk - updated_at INTEGER NOT NULL DEFAULT 0, -- newest-wins vs officer char entries - PRIMARY KEY (discord_id, day) -); - --- Officer-set per-CHARACTER availability. Raiders who never use the site --- (placeholder / unclaimed characters) cannot declare their own calendar, --- so officers set it from the raid planner. World-scoped (characters are --- per-server); character_name stored lower-cased. Where a player's own --- user_availability row ALSO exists, the NEWER edit wins (ties go to the --- player) — an officer must be able to correct a stale self-declaration, --- and a player re-declaring afterwards takes it back. 'available' is --- stored here (not deleted) so an officer can clear a stale player AFK. -CREATE TABLE IF NOT EXISTS character_availability ( - world TEXT NOT NULL, - character_name TEXT NOT NULL, -- lower-cased - day TEXT NOT NULL, -- ISO date YYYY-MM-DD - status TEXT NOT NULL, -- available / tentative / afk - set_by TEXT NOT NULL, -- officer discord id - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - PRIMARY KEY (world, character_name, day) -); - --- Saved AA planner builds. Owned by a user and pinned to the character they --- were planned from (the planner lives on the character page's AA tab). --- allocations is JSON {tree_id: {node_id: rank}}. share_slug is minted at --- creation so any plan can be linked read-only without a separate publish step. -CREATE TABLE IF NOT EXISTS aa_plans ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - discord_id TEXT NOT NULL REFERENCES users(discord_id) ON DELETE CASCADE, - world TEXT NOT NULL, - character_name TEXT NOT NULL, - name TEXT NOT NULL, - xpac TEXT, - allocations TEXT NOT NULL DEFAULT '{}', - share_slug TEXT NOT NULL UNIQUE, - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) -); - -CREATE INDEX IF NOT EXISTS idx_aa_plans_owner ON aa_plans(discord_id, world, character_name); - --- Per-Discord-guild registry linking a Discord server to an EQ2 guild. --- Configured via the bot's /lexicon command group (manage_guild gated). --- voice_channel_id is the raid voice channel polled by the Phase 3 --- voice-attendance cog. Brand-new table so CREATE TABLE IF NOT EXISTS is --- enough on existing DBs and no migration entry is needed. -CREATE TABLE IF NOT EXISTS discord_guild_links ( - discord_guild_id TEXT PRIMARY KEY, -- Discord snowflake as text - world TEXT NOT NULL, -- canonical world casing - guild_name TEXT NOT NULL, -- canonical Census casing - voice_channel_id TEXT, -- NULL = voice polling off - linked_by TEXT NOT NULL, -- discord id of the linker - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - parses_channel_id TEXT, -- NULL = parse posting off - parses_posted_at INTEGER NOT NULL DEFAULT 0 -- poster watermark (unix) -); - --- Officer corrections to derived attendance categories, per session per --- character. Applied LAST by the derivation (overrides beat observation --- evidence); clearing a row restores the derived truth. The observed --- record itself is never edited - corrections stay auditable + reversible. -CREATE TABLE IF NOT EXISTS attendance_overrides ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id INTEGER NOT NULL REFERENCES attendance_sessions(id) ON DELETE CASCADE, - character_name TEXT NOT NULL, - category TEXT NOT NULL, -- present/sat_out/afk/awol/absent - set_by TEXT NOT NULL, -- officer discord id - set_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - UNIQUE(session_id, character_name) -); - --- Officer-authored attendance timelines: a character's session split into --- timed periods (present/sat_out/afk) so a mid-raid bench shows as "sat out --- 21:00-22:00" instead of one flat category. Any manual rows for a character --- REPLACE that character's derived timeline in the derivation (the parser's --- observations stay untouched - reverting is deleting the rows). -CREATE TABLE IF NOT EXISTS attendance_segments ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id INTEGER NOT NULL REFERENCES attendance_sessions(id) ON DELETE CASCADE, - character_name TEXT NOT NULL, - category TEXT NOT NULL, -- present/sat_out/afk (timed states only) - started_at INTEGER NOT NULL, - ended_at INTEGER NOT NULL, - set_by TEXT NOT NULL, -- officer discord id - set_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) -); -CREATE INDEX IF NOT EXISTS idx_attendance_segments_session ON attendance_segments(session_id); - --- Per-guild feature switches editable by the guild LEADER (Census rank_id 0) --- or an admin. One boolean column per setting; an absent row means every --- default. Adding a setting later = ADD COLUMN in migrations.sql + a guard --- in migrations.py (enforced by _assertions.py). Column comments must not --- contain commas (the schema assertion splits column defs on them). -CREATE TABLE IF NOT EXISTS guild_settings ( - world TEXT NOT NULL, - guild_name TEXT NOT NULL, - officers_can_delete_parses INTEGER NOT NULL DEFAULT 1, -- 1 = officers may delete guild parses (today's behaviour) - updated_by TEXT, -- discord id of the leader/admin who last saved - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - PRIMARY KEY (world, guild_name) -); - --- Site-wide (NOT per-server) key/value settings edited from the admin page. --- Tiny by design: a handful of string knobs (e.g. discord_invite_url). A --- setting is "unset" when its row is absent. -CREATE TABLE IF NOT EXISTS site_settings ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - updated_by TEXT NOT NULL, -- admin discord id - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) -); - --- Guild recruitment profiles: officer-editable, publicly viewable. An --- absent row means "not recruiting" with an empty profile. Keyed by the --- census guild id so a profile (and its logo) survives a guild RENAME; --- guild_name is a display/lookup column refreshed on every officer save --- and by the daily census sweep (which also auto-delists guilds whose id --- no longer exists in census). The logo is a server-re-encoded 200x200-max --- WebP stored as a BLOB so it rides the users.db litestream replication --- and the erasure sweep. The JSON list columns are validated server-side --- against fixed catalogues (adventure class names / curated tag slugs / --- roster character names) before any write. Column comments must not --- contain commas (_assertions.py). -CREATE TABLE IF NOT EXISTS guild_recruitment ( - world TEXT NOT NULL, - guild_id INTEGER NOT NULL, -- census guild id (rename-proof anchor) - guild_name TEXT NOT NULL, -- current name verbatim casing (kept fresh) - recruiting INTEGER NOT NULL DEFAULT 0, -- 1 = listed on the browse page - description TEXT NOT NULL DEFAULT '', -- sanitised + blocklist-screened free text - classes_json TEXT NOT NULL DEFAULT '[]', -- JSON array of needed adventure class names - tags_json TEXT NOT NULL DEFAULT '[]', -- JSON array of curated tag slugs - contacts_json TEXT NOT NULL DEFAULT '[]', -- JSON array of in-game contact character names - discord_url TEXT, -- validated https discord invite or NULL - updated_by TEXT, -- discord id of the last profile save - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - logo BLOB, -- re-encoded WebP bytes or NULL - logo_media_type TEXT, -- image/webp when a logo is present - logo_uploaded_by TEXT, -- discord id of the logo uploader - logo_uploaded_at INTEGER, -- unix seconds (drives the ETag + cache busting) - PRIMARY KEY (world, guild_id) -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_guild_recruitment_name ON guild_recruitment(world, guild_name); -CREATE INDEX IF NOT EXISTS idx_guild_recruitment_listing ON guild_recruitment(world, recruiting); diff --git a/tests/server/test_db_migrations.py b/tests/server/test_db_migrations.py deleted file mode 100644 index 05a95a9b..00000000 --- a/tests/server/test_db_migrations.py +++ /dev/null @@ -1,319 +0,0 @@ -"""Memory [[test-migrations-against-old-db-shape]]: init_db must succeed -on a pre-migration-shape DB, not just a fresh one. The repo has crashed -in prod before because a column-dependent index in _SCHEMA referenced a -column not yet added on an existing DB. - -This test seeds a v1-shape DB by hand, then runs init_db and asserts -every modern column + index is present afterwards. -""" - -from __future__ import annotations - -import sqlite3 -from pathlib import Path -from unittest.mock import patch - -import pytest - -from backend.server import db as users_db -from backend.server.db._assertions import assert_schema_complete - - -@pytest.fixture -def v1_db(tmp_path: Path) -> Path: - """A user-table-only DB that mirrors the original schema BEFORE any - of the column-add migrations (users.access_status, - character_claims.world, character_claims.is_primary, etc.).""" - db = tmp_path / "users.db" - with sqlite3.connect(db) as conn: - conn.executescript(""" - CREATE TABLE users ( - discord_id TEXT PRIMARY KEY, - discord_name TEXT NOT NULL, - avatar TEXT, - first_seen INTEGER NOT NULL DEFAULT (strftime('%s','now')), - last_seen INTEGER NOT NULL DEFAULT (strftime('%s','now')) - ); - CREATE TABLE character_claims ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - discord_id TEXT NOT NULL, - character_name TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending' - ); - INSERT INTO users (discord_id, discord_name) VALUES ('123', 'OldUser'); - INSERT INTO character_claims (discord_id, character_name) VALUES ('123', 'Vortex'); - """) - return db - - -def test_init_db_migrates_v1_to_current(v1_db: Path) -> None: - """The v1-shape DB should upgrade cleanly to the current schema.""" - users_db.init_db(v1_db) - - with sqlite3.connect(v1_db) as conn: - # Modern columns exist on users. - users_cols = {row[1] for row in conn.execute("PRAGMA table_info(users)")} - assert "access_status" in users_cols - assert "discord_username" in users_cols - - # Modern columns exist on character_claims. - claims_cols = {row[1] for row in conn.execute("PRAGMA table_info(character_claims)")} - assert "world" in claims_cols - assert "is_primary" in claims_cols - - # The column-dependent indexes exist. - indexes = {row[1] for row in conn.execute("PRAGMA index_list(character_claims)")} - assert "idx_claims_world" in indexes - assert "idx_claims_primary" in indexes - - # BE-073: access_status index exists on users. - user_indexes = {row[1] for row in conn.execute("PRAGMA index_list(users)")} - assert "idx_users_access" in user_indexes - - # Existing data survived. - assert conn.execute("SELECT discord_name FROM users WHERE discord_id='123'").fetchone()[0] == "OldUser" - - -def test_init_db_on_fresh_db(tmp_path: Path) -> None: - """The fresh-DB path also works (no pre-existing tables).""" - db = tmp_path / "users.db" - users_db.init_db(db) - with sqlite3.connect(db) as conn: - # Both old and modern columns present. - users_cols = {row[1] for row in conn.execute("PRAGMA table_info(users)")} - assert "access_status" in users_cols - assert "discord_username" in users_cols - - claims_cols = {row[1] for row in conn.execute("PRAGMA table_info(character_claims)")} - assert "world" in claims_cols - assert "is_primary" in claims_cols - - # Indexes exist on fresh DB. - indexes = {row[1] for row in conn.execute("PRAGMA index_list(character_claims)")} - assert "idx_claims_world" in indexes - assert "idx_claims_primary" in indexes - - user_indexes = {row[1] for row in conn.execute("PRAGMA index_list(users)")} - assert "idx_users_access" in user_indexes - - -def test_init_db_idempotent(v1_db: Path) -> None: - """Running init_db twice in a row is safe — no double-ALTER errors.""" - users_db.init_db(v1_db) - users_db.init_db(v1_db) # must not raise - - -# --------------------------------------------------------------------------- -# BE-070: schema-vs-migrations drift assertion -# --------------------------------------------------------------------------- - - -def test_assert_schema_complete_passes_on_current_db(tmp_path: Path) -> None: - """assert_schema_complete must not raise after init_db produces a fully - migrated DB — every column in SCHEMA should be present.""" - db = tmp_path / "users.db" - users_db.init_db(db) - with sqlite3.connect(db) as conn: - assert_schema_complete(conn) # must not raise - - -def test_assert_schema_complete_fires_on_missing_column(tmp_path: Path) -> None: - """If SCHEMA declares a column that's absent from the live DB, the - assertion should fire immediately — rather than silently failing at - runtime when the column is first accessed. - - We simulate the drift by patching SCHEMA to include a fake column that - the migration never adds, then calling assert_schema_complete against a - freshly initialised (un-drifted) DB.""" - db = tmp_path / "users.db" - users_db.init_db(db) - - # Inject a fake column into SCHEMA's users table definition. - import backend.server.db.schema as _schema_mod - - drifted_schema = _schema_mod.SCHEMA.replace( - "CREATE TABLE IF NOT EXISTS users (", - "CREATE TABLE IF NOT EXISTS users (\n _fake_drift_column TEXT,", - ) - with ( - patch.object(_schema_mod, "SCHEMA", drifted_schema), - patch("backend.server.db._assertions.SCHEMA", drifted_schema), - ): - with sqlite3.connect(db) as conn: - with pytest.raises(AssertionError, match="_fake_drift_column"): - assert_schema_complete(conn) - - -def test_init_db_migrates_old_servers_table(tmp_path: Path) -> None: - """A pre-next_xpac servers table (the 2026-08 prod shape) gains the - upcoming-expansion columns on init_db, keeping its rows.""" - db = tmp_path / "users.db" - with sqlite3.connect(db) as conn: - conn.executescript(""" - CREATE TABLE servers ( - world TEXT PRIMARY KEY, - subdomain TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - max_level INTEGER NOT NULL, - current_xpac TEXT, - launch_dt TEXT, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) - ); - INSERT INTO servers (world, subdomain, display_name, max_level) - VALUES ('Varsoon', 'varsoon', 'Varsoon', 70); - """) - - users_db.init_db(db) - - with sqlite3.connect(db) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(servers)")} - assert {"is_default", "next_xpac", "next_xpac_dt"} <= cols - row = conn.execute("SELECT max_level, next_xpac, next_xpac_dt FROM servers WHERE world='Varsoon'").fetchone() - assert row == (70, None, None) # existing row survived, new cols NULL - - -def test_init_db_migrates_pre_placeholder_roles_table(tmp_path: Path) -> None: - """raid_roster_roles created before the 2026-09 placeholder-raider - migration (no placeholder/cls columns) must upgrade in place with its - rows intact.""" - db = tmp_path / "users.db" - with sqlite3.connect(db) as conn: - conn.executescript(""" - CREATE TABLE raid_roster_roles ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL, - guild_name TEXT NOT NULL, - character_name TEXT NOT NULL, - role TEXT NOT NULL, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_by TEXT NOT NULL, - UNIQUE(world, guild_name, character_name) - ); - INSERT INTO raid_roster_roles (world, guild_name, character_name, role, updated_by) - VALUES ('Varsoon', 'Exordium', 'Tanky', 'raider', 'u1'); - """) - users_db.init_db(db) - with sqlite3.connect(db) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(raid_roster_roles)")} - assert {"placeholder", "cls"} <= cols - row = conn.execute( - "SELECT role, placeholder, cls FROM raid_roster_roles WHERE character_name='Tanky'" - ).fetchone() - assert row == ("raider", 0, None) - - -def test_init_db_migrates_pre_rollover_servers_table(tmp_path: Path) -> None: - """servers created before the xpac-rollover stamp (no - current_xpac_started_dt) must upgrade in place with rows intact.""" - db = tmp_path / "users.db" - with sqlite3.connect(db) as conn: - conn.executescript(""" - CREATE TABLE servers ( - world TEXT PRIMARY KEY, - subdomain TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - max_level INTEGER NOT NULL, - current_xpac TEXT, - launch_dt TEXT, - next_xpac TEXT, - next_xpac_dt TEXT, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - is_default INTEGER NOT NULL DEFAULT 1 - ); - INSERT INTO servers (world, subdomain, display_name, max_level, current_xpac) - VALUES ('Wuoshi', 'wuoshi', 'Wuoshi', 70, 'EoF'); - """) - users_db.init_db(db) - with sqlite3.connect(db) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(servers)")} - assert "current_xpac_started_dt" in cols - row = conn.execute("SELECT current_xpac, current_xpac_started_dt FROM servers WHERE world='Wuoshi'").fetchone() - assert row == ("EoF", None) - - -def test_init_db_migrates_pre_parse_posting_links_table(tmp_path: Path) -> None: - """discord_guild_links created before the 2026-09 parse-posting - migration (no parses_channel_id/parses_posted_at) must upgrade in - place with its rows intact.""" - db = tmp_path / "users.db" - with sqlite3.connect(db) as conn: - conn.executescript(""" - CREATE TABLE discord_guild_links ( - discord_guild_id TEXT PRIMARY KEY, - world TEXT NOT NULL, - guild_name TEXT NOT NULL, - voice_channel_id TEXT, - linked_by TEXT NOT NULL, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) - ); - INSERT INTO discord_guild_links (discord_guild_id, world, guild_name, linked_by) - VALUES ('648253204760625160', 'Varsoon', 'Exordium', 'u1'); - """) - users_db.init_db(db) - with sqlite3.connect(db) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(discord_guild_links)")} - assert {"parses_channel_id", "parses_posted_at"} <= cols - row = conn.execute("SELECT guild_name, parses_channel_id, parses_posted_at FROM discord_guild_links").fetchone() - assert row == ("Exordium", None, 0) - - -def test_init_db_migrates_pre_stamp_user_availability_table(tmp_path: Path) -> None: - """user_availability created before the 2026-09 newest-wins migration - (no updated_at) must upgrade in place; legacy rows stamp 0 so any - stamped edit beats them.""" - db = tmp_path / "users.db" - with sqlite3.connect(db) as conn: - conn.executescript(""" - CREATE TABLE user_availability ( - discord_id TEXT NOT NULL, - day TEXT NOT NULL, - status TEXT NOT NULL, - PRIMARY KEY (discord_id, day) - ); - INSERT INTO user_availability (discord_id, day, status) VALUES ('u1', '2026-08-01', 'afk'); - """) - users_db.init_db(db) - with sqlite3.connect(db) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(user_availability)")} - assert "updated_at" in cols - row = conn.execute("SELECT status, updated_at FROM user_availability WHERE discord_id='u1'").fetchone() - assert row == ("afk", 0) - - -def test_init_db_rebuilds_name_keyed_guild_recruitment_table(tmp_path: Path) -> None: - """guild_recruitment's name-keyed first cut (dev-only, pre-release) is - rebuilt onto the census-guild-id PK. Rows can't be mapped onto the NOT - NULL id, so the rebuild starts empty — acceptable because that shape - never deployed. Both indexes must exist on the rebuilt table.""" - db = tmp_path / "users.db" - with sqlite3.connect(db) as conn: - conn.executescript(""" - CREATE TABLE guild_recruitment ( - world TEXT NOT NULL, - guild_name TEXT NOT NULL, - recruiting INTEGER NOT NULL DEFAULT 0, - description TEXT NOT NULL DEFAULT '', - classes_json TEXT NOT NULL DEFAULT '[]', - tags_json TEXT NOT NULL DEFAULT '[]', - contacts_json TEXT NOT NULL DEFAULT '[]', - discord_url TEXT, - updated_by TEXT, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - logo BLOB, - logo_media_type TEXT, - logo_uploaded_by TEXT, - logo_uploaded_at INTEGER, - PRIMARY KEY (world, guild_name) - ); - INSERT INTO guild_recruitment (world, guild_name, recruiting, description, updated_by) - VALUES ('Varsoon', 'Exordium', 1, 'dev-only throwaway row', 'u1'); - """) - users_db.init_db(db) - with sqlite3.connect(db) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(guild_recruitment)")} - assert "guild_id" in cols - pk_cols = [row[1] for row in conn.execute("PRAGMA table_info(guild_recruitment)") if row[5] > 0] - assert set(pk_cols) == {"world", "guild_id"} - assert conn.execute("SELECT COUNT(*) FROM guild_recruitment").fetchone()[0] == 0 - indexes = {row[1] for row in conn.execute("PRAGMA index_list(guild_recruitment)")} - assert {"idx_guild_recruitment_name", "idx_guild_recruitment_listing"} <= indexes From eea281b5d991517593ae642f5eb53790bbc72166 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:05:31 +0100 Subject: [PATCH 12/32] test(pg): serialise concurrent pytest sessions on the shared test database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit provision_for_session drops + rebuilds the session and scratch schemas, so two pytest invocations sharing one database clobbered each other (roving UndefinedTable failures — the old per-PID tmpdir layout was parallel-safe, TEST-039). A process-lifetime advisory lock now queues a second session until the first exits. Also: psycopg.sql via explicit import (stubs don't re-export it). Co-Authored-By: Claude Fable 5 --- tests/fixtures/pg.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/pg.py b/tests/fixtures/pg.py index 2217507a..775e8e96 100644 --- a/tests/fixtures/pg.py +++ b/tests/fixtures/pg.py @@ -30,6 +30,7 @@ import psycopg import pytest from psycopg import conninfo +from psycopg import sql as pgsql from psycopg.rows import dict_row from backend import pg, pg_migrate @@ -38,6 +39,15 @@ #: TEST_DATABASE_URL in the environment or .env. _DEFAULT_TEST_DSN = "postgresql://postgres:postgres@localhost:5432/eq2lexicon_test" +#: Advisory lock serialising pytest SESSIONS on one test database (distinct +#: from pg_migrate's lock). provision_for_session drops + rebuilds the +#: session and scratch schemas, which would clobber another in-flight run — +#: the old per-PID tmpdir layout made parallel invocations safe (TEST-039), +#: so queue them instead of racing. Held by a dedicated connection for the +#: LIFETIME of the process (released when the process exits). +_SESSION_LOCK_ID = 0x_E92_7E57 # spells-ish "eq2TEST" +_session_lock_conn: Any = None + _INSTALL_HINT = ( "The test suite needs a local PostgreSQL 17 server:\n" " winget install PostgreSQL.PostgreSQL.17\n" @@ -72,7 +82,7 @@ def provision_for_session() -> None: # via the maintenance DB (CREATE DATABASE can't run in a txn). admin = conninfo.make_conninfo(**{**params, "dbname": "postgres"}) with psycopg.connect(admin, autocommit=True, connect_timeout=3) as conn: - conn.execute(psycopg.sql.SQL("CREATE DATABASE {}").format(psycopg.sql.Identifier(dbname))) + conn.execute(pgsql.SQL("CREATE DATABASE {}").format(pgsql.Identifier(dbname))) except psycopg.OperationalError as exc: pytest.exit( f"Cannot reach the test PostgreSQL database {dbname!r}: {exc}\n{_INSTALL_HINT}", @@ -82,6 +92,16 @@ def provision_for_session() -> None: if os.environ.get("PYTEST_XDIST_WORKER"): return + # Queue behind any other pytest session on this database (see + # _SESSION_LOCK_ID). xdist workers never reach here, so the + # controller's lock covers its whole worker tree. + global _session_lock_conn + _session_lock_conn = psycopg.connect(test_dsn, autocommit=True) + got = _session_lock_conn.execute("SELECT pg_try_advisory_lock(%s)", (_SESSION_LOCK_ID,)).fetchone() + if not (got and got[0]): + print("[pg-fixtures] another pytest session holds the test database — waiting for it to finish...") + _session_lock_conn.execute("SELECT pg_advisory_lock(%s)", (_SESSION_LOCK_ID,)) + with pg.connection() as conn: # Drop the ledger so pg_migrate re-applies everything, then the # session schemas and any scratch schemas a crashed run left over. From 5c9a7852d5be38cac00134e59eda441b2fa112ec Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:19:20 +0100 Subject: [PATCH 13/32] =?UTF-8?q?test(pg):=20convert=20the=20users-touchin?= =?UTF-8?q?g=20test=20suite=20to=20the=20PG=20fixtures=20=E2=80=94=20full?= =?UTF-8?q?=20suite=20green?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 25 users-family test files now run on leased scratch schemas / pg_conn seeding (three parallel conversion agents + verification pass). Highlights: PG actually enforces the users FKs SQLite never did, so store tests seed their user rows; erasure tests drive the new users_schema= seam (parses half stays SQLite); servers/xpac tests adapt to the migration-seeded Varsoon+Wuoshi baseline; the legacy SQLite-ALTER regression tests became information_schema invariants; guild_context's missing-table degrade is simulated with a nonexistent schema. Route fix the conversion surfaced: attendance list/detail still json.loads'd jsonb zones/uploaders (TypeError 500 once non-empty) — now uses the parsed values. test_raid_live fixtures carry days as int[]; the role-requests 409 test raises UniqueViolation. Full suite: 2149 passed, 0 failed, on local PostgreSQL 17. Co-Authored-By: Claude Fable 5 --- backend/db_catalogue.py | 1 + backend/server/api/attendance.py | 7 +- tests/bot/test_guild_context.py | 26 +-- tests/server/test_aa_plans.py | 29 +-- tests/server/test_api_tokens.py | 28 +-- tests/server/test_attendance.py | 61 +++--- tests/server/test_attendance_retention.py | 36 ++-- tests/server/test_auth_tokens.py | 59 ++--- tests/server/test_character_rotation.py | 32 +++ tests/server/test_claim_per_server.py | 28 ++- tests/server/test_discord_links.py | 17 +- tests/server/test_downloads.py | 35 ++- tests/server/test_erasure.py | 143 ++++++------ tests/server/test_export.py | 30 ++- tests/server/test_favorites.py | 45 ++-- tests/server/test_guild_recruitment.py | 20 +- tests/server/test_item_watch_db.py | 57 ++--- tests/server/test_raid_live.py | 6 +- tests/server/test_raid_planning.py | 48 +++-- tests/server/test_raid_schedule.py | 41 ++-- tests/server/test_rankings.py | 26 +-- tests/server/test_role_requests.py | 4 +- tests/server/test_server_context.py | 44 ++-- tests/server/test_server_route.py | 26 +-- tests/server/test_servers_db.py | 251 ++++++++-------------- tests/server/test_site_settings.py | 16 +- tests/server/test_users_db_roles.py | 98 ++++----- tests/server/test_xpac_rollover.py | 18 +- 28 files changed, 564 insertions(+), 668 deletions(-) diff --git a/backend/db_catalogue.py b/backend/db_catalogue.py index a8f3ac14..f50b93d4 100644 --- a/backend/db_catalogue.py +++ b/backend/db_catalogue.py @@ -313,6 +313,7 @@ async def _db(self, *, row_factory: bool = False) -> AsyncIterator[aiosqlite.Con db.row_factory = aiosqlite.Row yield db + # --------------------------------------------------------------------------- # Postgres bases — the migrated families (users / parses / census / zones / # raids). One database, one schema per family; connections come from diff --git a/backend/server/api/attendance.py b/backend/server/api/attendance.py index a733593a..e7b59628 100644 --- a/backend/server/api/attendance.py +++ b/backend/server/api/attendance.py @@ -16,7 +16,6 @@ from __future__ import annotations import datetime as dt -import json import logging import sqlite3 from typing import cast @@ -474,7 +473,7 @@ async def list_attendance(request: Request, guild_name: str, limit: int = 25, be out.append( { **s, - "zones": json.loads(s["zones"] or "[]"), + "zones": s["zones"] or [], # jsonb — psycopg hands back the parsed list "scheduled": bool(s["scheduled"]), "counts": derive.session_counts(char_rows), } @@ -596,9 +595,9 @@ async def get_attendance_session(request: Request, guild_name: str, session_id: row["timeline_by"] = display.get(segs[0]["set_by"]) or segs[0]["set_by"] session["scheduled"] = bool(session["scheduled"]) - session["zones"] = json.loads(session["zones"] or "[]") + session["zones"] = session["zones"] or [] # jsonb — already parsed # Uploader discord ids are an audit detail — officers only. - session["uploaders"] = sorted(json.loads(session["uploaders"] or "{}")) if is_officer else None + session["uploaders"] = sorted(session["uploaders"] or {}) if is_officer else None return {"is_officer": is_officer, "session": session, "characters": char_rows, "users": user_rows} diff --git a/tests/bot/test_guild_context.py b/tests/bot/test_guild_context.py index 02acd050..4c0b5d6e 100644 --- a/tests/bot/test_guild_context.py +++ b/tests/bot/test_guild_context.py @@ -2,26 +2,18 @@ from __future__ import annotations -from pathlib import Path - import pytest from backend.bot.guild_context import FALLBACK_WORLD, resolve_guild_context -from backend.server.db import init_db from backend.server.db.discord_links import store as links_db from tests.fixtures.users_db import point_users_db_at -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema @pytest.mark.asyncio @@ -46,12 +38,12 @@ async def test_linked_guild_resolves(): @pytest.mark.asyncio -async def test_missing_table_degrades_to_fallback(tmp_path, monkeypatch): - """The bot may race the web lifespan's init_db on a fresh deploy — a +async def test_missing_table_degrades_to_fallback(monkeypatch): + """The bot may race the web lifespan's migrations on a fresh deploy — a missing registry table must resolve to the fallback, never crash.""" - empty = tmp_path / "empty.db" - empty.touch() - monkeypatch.setattr(links_db, "path", empty) + # A never-migrated schema: the registry SELECT raises UndefinedTable + # (a psycopg.Error subclass — the family guild_context degrades on). + point_users_db_at(monkeypatch, "users_schema_that_does_not_exist") ctx = await resolve_guild_context(42) assert (ctx.world, ctx.linked) == (FALLBACK_WORLD, False) diff --git a/tests/server/test_aa_plans.py b/tests/server/test_aa_plans.py index 3877456f..927eff50 100644 --- a/tests/server/test_aa_plans.py +++ b/tests/server/test_aa_plans.py @@ -1,38 +1,39 @@ """AA planner saved builds — DB layer + API tests. -Same harness as test_favorites.py: temp users.db via init_db + -point_users_db_at, signed session cookies against the global app fixture. +Same harness as test_favorites.py: isolated leased users schema via the +conftest ``users_schema`` fixture, signed session cookies against the +global app fixture. """ from __future__ import annotations import base64 import json -from pathlib import Path import itsdangerous import pytest from httpx import ASGITransport, AsyncClient -from backend.server.db import init_db from backend.server.db.aa_plans import store as aa_plans -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn _TEST_SECRET = "pytest-session-secret-not-real-0123456789" pytestmark = pytest.mark.asyncio -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``. Seeds the user rows the FK on + aa_plans.discord_id now actually enforces (SQLite never did).""" + with pg_conn(users_schema) as conn: + for uid in ("disc1", "disc2", "disc-1", "disc-2"): + conn.execute( + "INSERT INTO users (discord_id, discord_name) VALUES (%s, %s) ON CONFLICT DO NOTHING", + (uid, uid), + ) + return users_schema _ALLOC = {"42": {"101": 10, "102": 3}, "29": {"7": 1}} diff --git a/tests/server/test_api_tokens.py b/tests/server/test_api_tokens.py index bd5786d0..fc37a0a8 100644 --- a/tests/server/test_api_tokens.py +++ b/tests/server/test_api_tokens.py @@ -5,44 +5,36 @@ import pytest from backend.server.db.tokens import TokensStore +from tests.fixtures.pg import pg_conn @pytest.fixture -def tmp_users_db_for_coalesce(tmp_path): - """Minimal users.db with one approved user and one api token.""" - import sqlite3 as _sqlite3 - - from backend.server import db as users_db - - db_path = tmp_path / "users.db" - users_db.init_db(db_path) - with _sqlite3.connect(db_path) as conn: +def tmp_users_db_for_coalesce(users_schema: str) -> str: + """Isolated leased users schema with one approved user.""" + with pg_conn(users_schema) as conn: conn.execute( - "INSERT INTO users (discord_id, discord_name, access_status) VALUES (?, ?, ?)", + "INSERT INTO users (discord_id, discord_name, access_status) VALUES (%s, %s, %s)", ("user-coalesce", "Alice", "approved"), ) - conn.commit() - return db_path + return users_schema @pytest.mark.asyncio async def test_lookup_api_token_coalesces_writes(tmp_users_db_for_coalesce) -> None: """BE-011: two lookups within 60 s should issue at most one UPDATE.""" - from backend.server import db as users_db - - db_path = tmp_users_db_for_coalesce + schema = tmp_users_db_for_coalesce # Mint a token for the user. - raw, _row = await TokensStore(db_path).mint_api_token("user-coalesce", "Test Token") + raw, _row = await TokensStore(schema).mint_api_token("user-coalesce", "Test Token") # First lookup — should write last_used_at. - row1 = await TokensStore(db_path).lookup_api_token(raw) + row1 = await TokensStore(schema).lookup_api_token(raw) assert row1 is not None last_after_first = row1["last_used_at"] assert last_after_first is not None # Second lookup immediately after — within 60 s, so should NOT write. - row2 = await TokensStore(db_path).lookup_api_token(raw) + row2 = await TokensStore(schema).lookup_api_token(raw) assert row2 is not None # last_used_at returned by the second call is whatever's in the row; # since we didn't update, it equals what the first call wrote. diff --git a/tests/server/test_attendance.py b/tests/server/test_attendance.py index b70f1d6a..10388888 100644 --- a/tests/server/test_attendance.py +++ b/tests/server/test_attendance.py @@ -1,18 +1,17 @@ """Raid-attendance tests — store merge semantics, category derivation, routes. -Store tests run against a temp users.db (stores re-pointed by the autouse -fixture, same pattern as test_raid_planning.py). The ingest route reuses the -HMAC signing helpers from tests/server/_parses_ingest_fixtures.py so the -signature contract stays pinned by one source of truth; guild resolution and -the schedule probe are patched in the attendance module's namespace (the -route imports them by name). +Store tests run against a leased scratch Postgres schema (stores re-pointed +by the autouse fixture, same pattern as test_raid_planning.py). The ingest +route reuses the HMAC signing helpers from +tests/server/_parses_ingest_fixtures.py so the signature contract stays +pinned by one source of truth; guild resolution and the schedule probe are +patched in the attendance module's namespace (the route imports them by +name). """ from __future__ import annotations -import json import time -from pathlib import Path from unittest.mock import AsyncMock, patch import pytest @@ -20,14 +19,13 @@ from backend.server import attendance as derive from backend.server.attendance import derive_categories, session_counts -from backend.server.db import init_db from backend.server.db.attendance import ( MAX_SESSION_SPAN_S, MERGE_GAP_S, session_day_for, ) from backend.server.db.attendance import store as attendance_db -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn from tests.server._parses_ingest_fixtures import _fake_require_user, _signed_post_kwargs # --------------------------------------------------------------------------- @@ -35,30 +33,31 @@ # --------------------------------------------------------------------------- -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema @pytest.fixture(autouse=True) -def _grant_subscriber(users_db: Path): - """Attendance is in limited preview behind the 'subscriber' role — grant - it to the identities the route tests act as (the HMAC fixture's token - user + the guild-view session user). The gate's own tests use other - ids.""" - import sqlite3 - - with sqlite3.connect(users_db) as conn: +def _seed_users(users_db: str): + """Seed the Discord identities the tests write rows for — claims and + user_availability carry enforced FKs to users on Postgres (SQLite never + enabled the pragma). Also grant 'subscriber': attendance is in limited + preview behind that role — the HMAC fixture's token user + the + guild-view session user get it; the gate's own tests use other ids.""" + with pg_conn(users_db) as conn: + for did in ("discord-123", "member-1", "u-main", "u-afk", "u-voice", "u-swap", "u-pure"): + conn.execute( + "INSERT INTO users (discord_id, discord_name, access_status) " + "VALUES (%s, %s, 'approved') ON CONFLICT (discord_id) DO NOTHING", + (did, did), + ) for did in ("discord-123", "member-1"): conn.execute( - "INSERT OR IGNORE INTO user_roles (discord_id, role, granted_by) VALUES (?, 'subscriber', 'test')", + "INSERT INTO user_roles (discord_id, role, granted_by) VALUES (%s, 'subscriber', 'test') " + "ON CONFLICT DO NOTHING", (did,), ) @@ -128,7 +127,7 @@ async def test_create_then_read_back(): assert {(o["character_name"], o["kind"]) for o in obs} == {("Tanky", "raid"), ("Benchy", "online")} session = await attendance_db.get_session(res["session_id"]) assert session is not None - assert json.loads(session["zones"]) == ["VP"] + assert session["zones"] == ["VP"] # jsonb comes back parsed assert session["started_at"] == T0 and session["ended_at"] == T0 + 600 @@ -152,8 +151,8 @@ async def test_two_uploaders_merge_commutatively(low_window_floor): results[order] = ( session["started_at"], session["ended_at"], - json.loads(session["zones"]), - sorted(json.loads(session["uploaders"])), + session["zones"], + sorted(session["uploaders"]), # jsonb dict — sorted() iterates its keys [(o["character_name"], o["first_seen"], o["last_seen"]) for o in obs], ) await attendance_db.delete_session(r1["session_id"]) diff --git a/tests/server/test_attendance_retention.py b/tests/server/test_attendance_retention.py index 9efbf3d2..b712c602 100644 --- a/tests/server/test_attendance_retention.py +++ b/tests/server/test_attendance_retention.py @@ -3,40 +3,31 @@ from __future__ import annotations -import sqlite3 -from pathlib import Path - import pytest from backend.server.constants import VOICE_OBSERVATION_RETENTION_DAYS -from backend.server.db import init_db from backend.server.db.attendance import store as attendance_store -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn NOW = 1_800_000_000 DAY = 86400 -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema -def _seed(db: Path) -> None: - with sqlite3.connect(db) as c: +def _seed(schema: str) -> None: + with pg_conn(schema) as c: c.execute( "INSERT INTO attendance_sessions (world, guild_name, session_day, seq, started_at, ended_at) " - "VALUES ('Varsoon', 'Exordium', '2026-01-01', 0, ?, ?)", + "VALUES ('Varsoon', 'Exordium', '2026-01-01', 0, %s, %s)", (NOW - 200 * DAY, NOW - 200 * DAY + 3600), ) - sid = c.execute("SELECT id FROM attendance_sessions").fetchone()[0] + sid = c.execute("SELECT id FROM attendance_sessions").fetchone()["id"] rows = [ ("111", "voice", NOW - 200 * DAY), # stale voice → pruned ("222", "voice", NOW - 10 * DAY), # recent voice → kept @@ -46,7 +37,7 @@ def _seed(db: Path) -> None: for name, kind, seen in rows: c.execute( "INSERT INTO attendance_observations (session_id, character_name, kind, first_seen, last_seen) " - "VALUES (?, ?, ?, ?, ?)", + "VALUES (%s, %s, %s, %s, %s)", (sid, name, kind, seen, seen), ) @@ -55,8 +46,11 @@ async def test_prune_removes_only_stale_voice_rows(users_db): _seed(users_db) pruned = await attendance_store.prune_voice_observations(older_than_days=VOICE_OBSERVATION_RETENTION_DAYS, now=NOW) assert pruned == 1 - with sqlite3.connect(users_db) as c: - remaining = sorted(c.execute("SELECT character_name, kind FROM attendance_observations").fetchall()) + with pg_conn(users_db) as c: + remaining = sorted( + (r["character_name"], r["kind"]) + for r in c.execute("SELECT character_name, kind FROM attendance_observations").fetchall() + ) assert remaining == [("222", "voice"), ("Alt", "online"), ("Sihtric", "raid")] # Second sweep is a no-op. assert ( diff --git a/tests/server/test_auth_tokens.py b/tests/server/test_auth_tokens.py index b01bf0a9..dc7aa49d 100644 --- a/tests/server/test_auth_tokens.py +++ b/tests/server/test_auth_tokens.py @@ -4,10 +4,11 @@ from unittest.mock import AsyncMock, patch +import psycopg import pytest from httpx import ASGITransport, AsyncClient -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn def _fake_session_user(request=None) -> dict: @@ -150,28 +151,21 @@ async def test_revoke_404_when_token_missing_or_not_yours(app): # --------------------------------------------------------------------------- -# DB-layer unit tests (use a temp file DB) +# DB-layer unit tests (use an isolated leased schema) # --------------------------------------------------------------------------- @pytest.fixture -def tmp_users_db(tmp_path, monkeypatch): - db_path = tmp_path / "users.db" - from backend.server import db as users_db - - # init_db creates schema; point every domain store at the temp path. - users_db.init_db(db_path) - point_users_db_at(monkeypatch, db_path) - # Seed a user row so the FK on api_tokens.user_id is satisfied. - import sqlite3 as _sqlite3 - - with _sqlite3.connect(db_path) as conn: +def tmp_users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``) — every + store already points at it. Seed a user row so the FK on + api_tokens.user_id is satisfied (enforced on Postgres).""" + with pg_conn(users_schema) as conn: conn.execute( - "INSERT INTO users (discord_id, discord_name, access_status) VALUES (?, ?, ?)", + "INSERT INTO users (discord_id, discord_name, access_status) VALUES (%s, %s, %s)", ("user-123", "Alice", "approved"), ) - conn.commit() - return db_path + return users_schema @pytest.mark.asyncio @@ -209,16 +203,24 @@ async def test_db_lookup_rejects_garbage(tmp_users_db): @pytest.mark.asyncio async def test_db_lookup_survives_failed_last_used_touch(tmp_users_db, monkeypatch): - """The last-used bump is COSMETIC — a busy users.db there must never - fail token auth (live 2026-09-12: a raid-night upload burst hit + """The last-used bump is COSMETIC — a busy DB there must never fail + token auth (live 2026-09-12: a raid-night upload burst hit 'database is locked' on the touch and 500d /attendance/ingest).""" from backend.server import db as users_db from backend.server.db import tokens as tokens_mod raw, _ = await users_db.mint_api_token("user-123", "First") - # Force the touch to blow up with OperationalError (same family the - # locked-database failure raises). - monkeypatch.setitem(tokens_mod._SQL, "update_last_used_at", "UPDATE no_such_table SET x = ? WHERE y = ?") + # Force the touch to blow up with OperationalError (the transient + # connection-trouble family the store deliberately swallows). + touch_sql = tokens_mod._SQL["update_last_used_at"] + orig_execute = psycopg.AsyncConnection.execute + + async def _exec(self, query, *args, **kwargs): + if query == touch_sql: + raise psycopg.OperationalError("simulated connection failure during touch") + return await orig_execute(self, query, *args, **kwargs) + + monkeypatch.setattr(psycopg.AsyncConnection, "execute", _exec) found = await users_db.lookup_api_token(raw) assert found is not None @@ -229,16 +231,13 @@ async def test_db_lookup_survives_failed_last_used_touch(tmp_users_db, monkeypat @pytest.mark.asyncio async def test_db_revoke_scoped_to_user(tmp_users_db): """A user can't revoke another user's token.""" - import sqlite3 as _sqlite3 - from backend.server import db as users_db - with _sqlite3.connect(tmp_users_db) as conn: + with pg_conn(tmp_users_db) as conn: conn.execute( - "INSERT INTO users (discord_id, discord_name, access_status) VALUES (?, ?, ?)", + "INSERT INTO users (discord_id, discord_name, access_status) VALUES (%s, %s, %s)", ("user-456", "Bob", "approved"), ) - conn.commit() raw_alice, row_alice = await users_db.mint_api_token("user-123", "Alice's") # Bob tries to revoke Alice's token @@ -339,15 +338,17 @@ async def test_whoami_returns_empty_allowed_servers_when_unset(app): @pytest.mark.asyncio -async def test_whoami_returns_static_roles(app): +async def test_whoami_returns_static_roles(app, users_schema): """static_roles carries DB-granted roles — EQ2Parser reads it (plus is_admin) to decide whether to show its Raid tab (the attendance limited-preview gate).""" + from backend.server import db as users_db + + # user_roles.discord_id FK is enforced on Postgres — seed the user row. + await users_db.upsert_user(discord_id="user-non-admin", discord_name="alice", discord_username="alice", avatar=None) with patch("backend.server.api.auth_tokens.require_user_session_or_token", _fake_token_user): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: before = (await client.get("/api/auth/whoami")).json() - from backend.server import db as users_db - await users_db.grant_role("user-non-admin", "subscriber", granted_by="test") after = (await client.get("/api/auth/whoami")).json() assert before["static_roles"] == [] diff --git a/tests/server/test_character_rotation.py b/tests/server/test_character_rotation.py index a8b8c769..684cf02d 100644 --- a/tests/server/test_character_rotation.py +++ b/tests/server/test_character_rotation.py @@ -568,6 +568,38 @@ async def test_rotation_acceleration_strike_static_base(app): assert a["has_unparsed_damage"] is False +@pytest.mark.asyncio +async def test_rotation_exorcise_pulse_interval_measured(app): + """Exorcise's maintained stream ticks every 7.1s in game vs the census + text's 6s. The correction is keyed by spell name — the AA lives in the + shared CLERIC class tree, so Templar and Inquisitor both get it.""" + from backend.server.api.character import rotation as mod + + row = _row(903, "Exorcise", spell_type="spells", level=70, crc=890, beneficial=1) + row["tier"] = 1 + row["target_type"] = "self" + row["effects"] = json.dumps( + [ + {"description": "Applies Exorcise instantly and every 6 seconds.", "indentation": 0}, + {"description": "Inflicts 269 - 448 divine damage on targets in Area of Effect.", "indentation": 1}, + ] + ) + tree = {"name": "Cleric", "tree_type": "class", "nodes": [{"node_id": 100, "name": "Exorcise", "spellcrc": 890}]} + p1, p2, p3 = _catalogue_patches({}) + with ( + p1, + p2, + p3, + patch.object(mod._aas, "get_tree", lambda tid: tree), + patch.object(mod._spells, "find_by_crc_bands", lambda crc, tier: [row] if crc == 890 else []), + ): + r = await _get(app, _fake_char(spell_ids=[]), aa_trees=[(3, {"100": 1})]) + (a,) = r.json()["abilities"] + dots = [c for c in a["components"] if c["kind"] == "dot"] + assert dots and all(c["interval_s"] == 7.1 and c["from_pulse"] for c in dots) + assert a["maintained"] is True # self pulse, no duration → toggle + + @pytest.mark.asyncio async def test_rotation_lifeburn_static_base_with_per_hp(app): """Lifeburn: census bands are era-drifted; the static base carries the diff --git a/tests/server/test_claim_per_server.py b/tests/server/test_claim_per_server.py index cb402617..2a8a4d08 100644 --- a/tests/server/test_claim_per_server.py +++ b/tests/server/test_claim_per_server.py @@ -1,8 +1,8 @@ """Per-server claim isolation tests (Task 6 — scope claims per (user, world)). Tests use the DB helpers directly with explicit ``world`` args so they run -without a live HTTP layer and stay fast. The conftest plants a temp DB and -calls init_db() for us. +without a live HTTP layer and stay fast. Each test gets an isolated leased +users schema (the autouse ``users_db`` alias of ``users_schema`` below). Convention mirrors the rest of tests/web/: use the shared ``app`` fixture from conftest when testing route-level behaviour (x-server header), and call @@ -13,7 +13,6 @@ import pytest -from backend.server import db from backend.server.db import ( get_active_claims, review_claim, @@ -22,12 +21,18 @@ upsert_user, withdraw_claim, ) +from tests.fixtures.pg import pg_conn # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -_PATH = db.DB_PATH # redirected to the pytest tmpdir by conftest.py + +@pytest.fixture(autouse=True) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema async def _seed_user(discord_id: str) -> None: @@ -45,12 +50,15 @@ async def _seed_user(discord_id: str) -> None: # --------------------------------------------------------------------------- -def test_world_column_exists_in_schema(): - """init_db (run by conftest) must have added the world column.""" - import sqlite3 - - with sqlite3.connect(_PATH) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(character_claims)")} +def test_world_column_exists_in_schema(users_db): + """The users migration must have added the world column.""" + with pg_conn(users_db) as conn: + rows = conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'character_claims'", + (users_db,), + ).fetchall() + cols = {r["column_name"] for r in rows} assert "world" in cols, "world column not found in character_claims" diff --git a/tests/server/test_discord_links.py b/tests/server/test_discord_links.py index 1875a8ae..0204cdf7 100644 --- a/tests/server/test_discord_links.py +++ b/tests/server/test_discord_links.py @@ -2,25 +2,16 @@ from __future__ import annotations -from pathlib import Path - import pytest -from backend.server.db import init_db from backend.server.db.discord_links import store as links_db -from tests.fixtures.users_db import point_users_db_at - - -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema _GID = "648253204760625160" diff --git a/tests/server/test_downloads.py b/tests/server/test_downloads.py index c6021a4c..9ca38cff 100644 --- a/tests/server/test_downloads.py +++ b/tests/server/test_downloads.py @@ -1,45 +1,44 @@ """Download-counter DB layer + API tests. -DB layer runs against a temp users.db (which also proves the download_events -schema block passes _assertions.py via init_db). The API is tested with the -``app`` fixture: a signed session cookie for auth + mocked db helpers (same -pattern as test_favorites.py). +DB layer runs against an isolated leased users schema (conftest +``users_schema``). The API is tested with the ``app`` fixture: a signed +session cookie for auth + mocked db helpers (same pattern as +test_favorites.py). """ from __future__ import annotations import base64 import json -from pathlib import Path from unittest.mock import AsyncMock, patch import itsdangerous import pytest from httpx import ASGITransport, AsyncClient -from backend.server.db import init_db from backend.server.db.downloads import store as dl -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn _TEST_SECRET = "pytest-session-secret-not-real-0123456789" # --------------------------------------------------------------------------- -# DB layer (temp users.db) +# DB layer (isolated leased schema) # --------------------------------------------------------------------------- -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) # creates download_events (+ asserts schema completeness) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - """Point users.db (constant + every domain store) at this test's temp DB.""" - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``. Seeds the user rows the FK on + download_events.discord_id now actually enforces (SQLite never did).""" + with pg_conn(users_schema) as conn: + for uid in ("disc1", "disc2"): + conn.execute( + "INSERT INTO users (discord_id, discord_name) VALUES (%s, %s) ON CONFLICT DO NOTHING", + (uid, uid), + ) + return users_schema async def test_record_is_idempotent_per_user(users_db): diff --git a/tests/server/test_erasure.py b/tests/server/test_erasure.py index 230653e1..fb4c1b86 100644 --- a/tests/server/test_erasure.py +++ b/tests/server/test_erasure.py @@ -15,12 +15,12 @@ import itsdangerous import pytest from httpx import ASGITransport, AsyncClient +from psycopg.types.json import Json -from backend.server.db import init_db from backend.server.db.erasure import DELETED_SOURCE_DSN, DELETED_USER_ID, erase_user_sync from backend.server.parses import db as pdb +from tests.fixtures.pg import pg_conn from tests.fixtures.users import make_fake_admin -from tests.fixtures.users_db import point_users_db_at _TEST_SECRET = "pytest-session-secret-not-real-0123456789" VICTIM = "111" @@ -28,11 +28,11 @@ NOW = 1_800_000_000 -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db +@pytest.fixture(autouse=True) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema @pytest.fixture @@ -43,79 +43,76 @@ def parses_db(tmp_path, monkeypatch) -> Path: return db -@pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) - - -def _seed_users(db: Path) -> None: - with sqlite3.connect(db) as c: +def _seed_users(schema: str) -> None: + with pg_conn(schema) as c: for uid, name in ((VICTIM, "Victim"), (OTHER, "Other")): c.execute( "INSERT INTO users (discord_id, discord_name, discord_username, first_seen, last_seen, access_status) " - "VALUES (?, ?, ?, ?, ?, 'approved')", + "VALUES (%s, %s, %s, %s, %s, 'approved')", (uid, name, name.lower(), NOW, NOW), ) c.execute( - "INSERT INTO api_tokens (user_id, name, token_hash, token_prefix, created_at) VALUES (?, 'act', 'h1', 'eq2c_aaaa', ?)", + "INSERT INTO api_tokens (user_id, name, token_hash, token_prefix, created_at) VALUES (%s, 'act', 'h1', 'eq2c_aaaa', %s)", (VICTIM, NOW), ) c.execute( - "INSERT INTO api_tokens (user_id, name, token_hash, token_prefix, created_at) VALUES (?, 'act', 'h2', 'eq2c_bbbb', ?)", + "INSERT INTO api_tokens (user_id, name, token_hash, token_prefix, created_at) VALUES (%s, 'act', 'h2', 'eq2c_bbbb', %s)", (OTHER, NOW), ) - c.execute("INSERT INTO user_roles (discord_id, role, granted_by) VALUES (?, 'supporter', ?)", (VICTIM, OTHER)) - c.execute("INSERT INTO user_roles (discord_id, role, granted_by) VALUES (?, 'contributor', ?)", (OTHER, VICTIM)) + c.execute("INSERT INTO user_roles (discord_id, role, granted_by) VALUES (%s, 'supporter', %s)", (VICTIM, OTHER)) + c.execute( + "INSERT INTO user_roles (discord_id, role, granted_by) VALUES (%s, 'contributor', %s)", (OTHER, VICTIM) + ) c.execute( "INSERT INTO character_claims (discord_id, character_name, status, requested_at, reviewed_by, world) " - "VALUES (?, 'Sihtric', 'approved', ?, ?, 'Varsoon')", + "VALUES (%s, 'Sihtric', 'approved', %s, %s, 'Varsoon')", (VICTIM, NOW, OTHER), ) c.execute( "INSERT INTO character_claims (discord_id, character_name, status, requested_at, reviewed_by, world) " - "VALUES (?, 'Alt', 'approved', ?, ?, 'Varsoon')", + "VALUES (%s, 'Alt', 'approved', %s, %s, 'Varsoon')", (OTHER, NOW, VICTIM), ) c.execute( - "INSERT INTO character_favorites (discord_id, character_name, world) VALUES (?, 'Alt', 'Varsoon')", + "INSERT INTO character_favorites (discord_id, character_name, world) VALUES (%s, 'Alt', 'Varsoon')", (VICTIM,), ) - c.execute("INSERT INTO download_events (discord_id, slug) VALUES (?, 'act-plugin')", (VICTIM,)) + c.execute("INSERT INTO download_events (discord_id, slug) VALUES (%s, 'act-plugin')", (VICTIM,)) c.execute( "INSERT INTO attendance_sessions (world, guild_name, session_day, seq, started_at, ended_at, uploaders) " - "VALUES ('Varsoon', 'Exordium', '2026-09-20', 0, ?, ?, ?)", - (NOW, NOW + 3600, json.dumps({VICTIM: NOW, OTHER: NOW})), + "VALUES ('Varsoon', 'Exordium', '2026-09-20', 0, %s, %s, %s)", + (NOW, NOW + 3600, Json({VICTIM: NOW, OTHER: NOW})), ) - sid = c.execute("SELECT id FROM attendance_sessions").fetchone()[0] + sid = c.execute("SELECT id FROM attendance_sessions").fetchone()["id"] c.execute( "INSERT INTO attendance_observations (session_id, character_name, kind, first_seen, last_seen) " - "VALUES (?, ?, 'voice', ?, ?)", + "VALUES (%s, %s, 'voice', %s, %s)", (sid, VICTIM, NOW, NOW), ) c.execute( "INSERT INTO attendance_observations (session_id, character_name, kind, first_seen, last_seen) " - "VALUES (?, ?, 'voice', ?, ?)", + "VALUES (%s, %s, 'voice', %s, %s)", (sid, OTHER, NOW, NOW), ) c.execute( "INSERT INTO attendance_observations (session_id, character_name, kind, first_seen, last_seen) " - "VALUES (?, 'Sihtric', 'raid', ?, ?)", + "VALUES (%s, 'Sihtric', 'raid', %s, %s)", (sid, NOW, NOW), ) c.execute( - "INSERT INTO attendance_overrides (session_id, character_name, category, set_by) VALUES (?, 'Alt', 'afk', ?)", + "INSERT INTO attendance_overrides (session_id, character_name, category, set_by) VALUES (%s, 'Alt', 'afk', %s)", (sid, VICTIM), ) c.execute( "INSERT INTO guild_settings (world, guild_name, officers_can_delete_parses, updated_by) " - "VALUES ('Varsoon', 'Exordium', 0, ?)", + "VALUES ('Varsoon', 'Exordium', 0, %s)", (VICTIM,), ) c.execute( "INSERT INTO guild_recruitment (world, guild_id, guild_name, recruiting, description, updated_by, " "logo, logo_media_type, logo_uploaded_by, logo_uploaded_at) " - "VALUES ('Varsoon', 42, 'Exordium', 1, 'We raid.', ?, X'01', 'image/webp', ?, ?)", - (VICTIM, VICTIM, NOW), + "VALUES ('Varsoon', 42, 'Exordium', 1, 'We raid.', %s, %s, 'image/webp', %s, %s)", + (VICTIM, b"\x01", VICTIM, NOW), ) @@ -149,10 +146,17 @@ def _seed_parses(db: Path) -> None: def _count(db: Path, sql: str, *params) -> int: + """parses.db (still SQLite) — ``?`` placeholders.""" with sqlite3.connect(db) as c: return c.execute(sql, params).fetchone()[0] +def _count_u(schema: str, sql: str, *params) -> int: + """users schema (Postgres) — ``%s`` placeholders, COUNT aliased AS n.""" + with pg_conn(schema) as c: + return c.execute(sql, params).fetchone()["n"] + + # --------------------------------------------------------------------------- # Store function # --------------------------------------------------------------------------- @@ -162,42 +166,47 @@ def test_erase_removes_owned_rows_tombstones_authorship_and_strips_uploads(users _seed_users(users_db) _seed_parses(parses_db) - result = erase_user_sync(VICTIM, users_path=users_db, parses_path=parses_db, now=NOW) + result = erase_user_sync(VICTIM, users_schema=users_db, parses_path=parses_db, now=NOW) assert result.found is True # Owned rows gone; the other user's rows intact. - assert _count(users_db, "SELECT COUNT(*) FROM users WHERE discord_id = ?", VICTIM) == 0 - assert _count(users_db, "SELECT COUNT(*) FROM users WHERE discord_id = ?", OTHER) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM api_tokens WHERE user_id = ?", VICTIM) == 0 - assert _count(users_db, "SELECT COUNT(*) FROM api_tokens WHERE user_id = ?", OTHER) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM user_roles WHERE discord_id = ?", VICTIM) == 0 - assert _count(users_db, "SELECT COUNT(*) FROM character_claims WHERE discord_id = ?", VICTIM) == 0 - assert _count(users_db, "SELECT COUNT(*) FROM character_claims WHERE discord_id = ?", OTHER) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM character_favorites WHERE discord_id = ?", VICTIM) == 0 - assert _count(users_db, "SELECT COUNT(*) FROM download_events WHERE discord_id = ?", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", OTHER) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM api_tokens WHERE user_id = %s", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM api_tokens WHERE user_id = %s", OTHER) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM user_roles WHERE discord_id = %s", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM character_claims WHERE discord_id = %s", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM character_claims WHERE discord_id = %s", OTHER) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM character_favorites WHERE discord_id = %s", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM download_events WHERE discord_id = %s", VICTIM) == 0 # Authorship tombstoned to the placeholder row, which now exists. - assert _count(users_db, "SELECT COUNT(*) FROM users WHERE discord_id = ?", DELETED_USER_ID) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM character_claims WHERE reviewed_by = ?", VICTIM) == 0 - assert _count(users_db, "SELECT COUNT(*) FROM character_claims WHERE reviewed_by = ?", DELETED_USER_ID) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM user_roles WHERE granted_by = ?", DELETED_USER_ID) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM attendance_overrides WHERE set_by = ?", DELETED_USER_ID) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM guild_settings WHERE updated_by = ?", DELETED_USER_ID) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", DELETED_USER_ID) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM character_claims WHERE reviewed_by = %s", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM character_claims WHERE reviewed_by = %s", DELETED_USER_ID) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM user_roles WHERE granted_by = %s", DELETED_USER_ID) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM attendance_overrides WHERE set_by = %s", DELETED_USER_ID) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM guild_settings WHERE updated_by = %s", DELETED_USER_ID) == 1 # Recruitment: both author columns tombstoned; the guild's profile text # and logo blob stay (guild assets, not the person's data). - assert _count(users_db, "SELECT COUNT(*) FROM guild_recruitment WHERE updated_by = ?", DELETED_USER_ID) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM guild_recruitment WHERE logo_uploaded_by = ?", DELETED_USER_ID) == 1 - with sqlite3.connect(users_db) as c: - logo, desc = c.execute("SELECT logo, description FROM guild_recruitment").fetchone() - assert logo == b"\x01" and desc == "We raid." + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM guild_recruitment WHERE updated_by = %s", DELETED_USER_ID) == 1 + assert ( + _count_u(users_db, "SELECT COUNT(*) AS n FROM guild_recruitment WHERE logo_uploaded_by = %s", DELETED_USER_ID) + == 1 + ) + with pg_conn(users_db) as c: + row = c.execute("SELECT logo, description FROM guild_recruitment").fetchone() + assert bytes(row["logo"]) == b"\x01" and row["description"] == "We raid." assert result.tombstoned["character_claims.reviewed_by"] == 1 # Voice rows for the victim gone, the other person's and the raid row kept. - assert _count(users_db, "SELECT COUNT(*) FROM attendance_observations WHERE character_name = ?", VICTIM) == 0 - assert _count(users_db, "SELECT COUNT(*) FROM attendance_observations WHERE character_name = ?", OTHER) == 1 - assert _count(users_db, "SELECT COUNT(*) FROM attendance_observations WHERE kind = 'raid'") == 1 + assert ( + _count_u(users_db, "SELECT COUNT(*) AS n FROM attendance_observations WHERE character_name = %s", VICTIM) == 0 + ) + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM attendance_observations WHERE character_name = %s", OTHER) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM attendance_observations WHERE kind = 'raid'") == 1 assert result.voice_observations_deleted == 1 - # Uploaders audit scrubbed. - with sqlite3.connect(users_db) as c: - uploaders = json.loads(c.execute("SELECT uploaders FROM attendance_sessions").fetchone()[0]) + # Uploaders audit scrubbed (jsonb — comes back parsed). + with pg_conn(users_db) as c: + uploaders = c.execute("SELECT uploaders FROM attendance_sessions").fetchone()["uploaders"] assert uploaders == {OTHER: NOW} assert result.sessions_scrubbed == 1 # parses.db: uploads stay, identity stripped; hidden_by cleared; reports gone. @@ -215,11 +224,11 @@ def test_erase_removes_owned_rows_tombstones_authorship_and_strips_uploads(users def test_erase_is_idempotent_and_never_touches_the_tombstone(users_db, parses_db): _seed_users(users_db) - assert erase_user_sync(VICTIM, users_path=users_db, parses_path=parses_db, now=NOW).found is True - again = erase_user_sync(VICTIM, users_path=users_db, parses_path=parses_db, now=NOW) + assert erase_user_sync(VICTIM, users_schema=users_db, parses_path=parses_db, now=NOW).found is True + again = erase_user_sync(VICTIM, users_schema=users_db, parses_path=parses_db, now=NOW) assert again.found is False - assert erase_user_sync(DELETED_USER_ID, users_path=users_db, parses_path=parses_db).found is False - assert _count(users_db, "SELECT COUNT(*) FROM users WHERE discord_id = ?", DELETED_USER_ID) == 1 + assert erase_user_sync(DELETED_USER_ID, users_schema=users_db, parses_path=parses_db).found is False + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", DELETED_USER_ID) == 1 # --------------------------------------------------------------------------- @@ -253,7 +262,7 @@ async def test_admin_erase_route(app, users_db, parses_db): assert selfie.status_code == 400 assert audit.call_args.args[0] == "user_erased" assert audit.call_args.kwargs["discord_id"] == VICTIM - assert _count(users_db, "SELECT COUNT(*) FROM users WHERE discord_id = ?", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", VICTIM) == 0 async def test_admin_erase_requires_admin(app): @@ -270,7 +279,7 @@ async def test_self_service_erase_requires_typed_username_and_clears_session(app async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test", cookies=cookies) as client: wrong = await client.request("DELETE", "/api/auth/me", json={"confirm": "someone-else"}) assert wrong.status_code == 400 - assert _count(users_db, "SELECT COUNT(*) FROM users WHERE discord_id = ?", VICTIM) == 1 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", VICTIM) == 1 ok = await client.request("DELETE", "/api/auth/me", json={"confirm": " Victim "}) assert ok.status_code == 200 assert ok.json()["ok"] is True @@ -280,7 +289,7 @@ async def test_self_service_erase_requires_typed_username_and_clears_session(app assert "session=null" in set_cookie and "1970" in set_cookie assert audit.call_args.args[0] == "user_erased" assert audit.call_args.kwargs["self_service"] is True - assert _count(users_db, "SELECT COUNT(*) FROM users WHERE discord_id = ?", VICTIM) == 0 + assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", VICTIM) == 0 assert _count(parses_db, "SELECT COUNT(*) FROM encounters WHERE source_dsn = ?", DELETED_SOURCE_DSN) == 1 diff --git a/tests/server/test_export.py b/tests/server/test_export.py index 06f2ecc8..4b094144 100644 --- a/tests/server/test_export.py +++ b/tests/server/test_export.py @@ -13,8 +13,7 @@ import pytest from httpx import ASGITransport, AsyncClient -from backend.server.db import init_db -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn from tests.server._parses_ingest_fixtures import _fake_require_user, _minimal_payload, _signed_post_kwargs # --------------------------------------------------------------------------- @@ -22,16 +21,11 @@ # --------------------------------------------------------------------------- -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema @pytest.fixture @@ -48,12 +42,16 @@ def isolated_parses_db(tmp_path, monkeypatch: pytest.MonkeyPatch) -> Path: return p -def _grant_api(users_db: Path, discord_id: str = "discord-123") -> None: - import sqlite3 - - with sqlite3.connect(users_db) as conn: +def _grant_api(users_db: str, discord_id: str = "discord-123") -> None: + with pg_conn(users_db) as conn: + # user_roles.discord_id FK is enforced on Postgres — seed the user row. + conn.execute( + "INSERT INTO users (discord_id, discord_name, access_status) VALUES (%s, %s, 'approved') " + "ON CONFLICT DO NOTHING", + (discord_id, discord_id), + ) conn.execute( - "INSERT OR IGNORE INTO user_roles (discord_id, role, granted_by) VALUES (?, 'api', 'test')", + "INSERT INTO user_roles (discord_id, role, granted_by) VALUES (%s, 'api', 'test') ON CONFLICT DO NOTHING", (discord_id,), ) diff --git a/tests/server/test_favorites.py b/tests/server/test_favorites.py index dc41e4ba..e3c10b52 100644 --- a/tests/server/test_favorites.py +++ b/tests/server/test_favorites.py @@ -1,16 +1,15 @@ """Character-favourites DB layer + API tests. -DB layer is exercised against a temp users.db (explicit ``path=``) — this also -proves the new schema block passes ``_assertions.py`` via ``init_db``. The API -is tested with the ``app`` fixture: a signed session cookie for auth + mocked -db helpers (same pattern as test_raid_schedule.py). +DB layer is exercised against an isolated leased users schema (conftest +``users_schema``). The API is tested with the ``app`` fixture: a signed +session cookie for auth + mocked db helpers (same pattern as +test_raid_schedule.py). """ from __future__ import annotations import base64 import json -from pathlib import Path from unittest.mock import AsyncMock, patch import itsdangerous @@ -18,9 +17,9 @@ from httpx import ASGITransport, AsyncClient from backend.server.cache import favorite_count_cache -from backend.server.db import init_db, review_claim, submit_claim +from backend.server.db import review_claim, submit_claim from backend.server.db.favorites import store as fav -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn _TEST_SECRET = "pytest-session-secret-not-real-0123456789" @@ -35,21 +34,22 @@ def _clear_count_cache(): # --------------------------------------------------------------------------- -# DB layer (temp users.db) +# DB layer (isolated leased schema) # --------------------------------------------------------------------------- -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) # creates character_favorites (+ asserts schema completeness) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - """Point users.db (constant + every domain store) at this test's temp DB.""" - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``. Seeds the user rows the FK on + character_favorites.discord_id now actually enforces (SQLite never did).""" + with pg_conn(users_schema) as conn: + for uid in ("disc1", "disc2", "disc3"): + conn.execute( + "INSERT INTO users (discord_id, discord_name) VALUES (%s, %s) ON CONFLICT DO NOTHING", + (uid, uid), + ) + return users_schema async def test_add_remove_round_trip(users_db): @@ -100,15 +100,12 @@ async def test_world_scoping(users_db): async def test_list_newest_first(users_db): - import aiosqlite - await fav.add_favorite("disc1", "Alpha", "Varsoon", cap=_CAP) await fav.add_favorite("disc1", "Bravo", "Varsoon", cap=_CAP) # Force distinct created_at so ordering is deterministic. - async with aiosqlite.connect(users_db) as db: - await db.execute("UPDATE character_favorites SET created_at = 100 WHERE character_name = 'Alpha'") - await db.execute("UPDATE character_favorites SET created_at = 200 WHERE character_name = 'Bravo'") - await db.commit() + with pg_conn(users_db) as conn: + conn.execute("UPDATE character_favorites SET created_at = 100 WHERE character_name = 'Alpha'") + conn.execute("UPDATE character_favorites SET created_at = 200 WHERE character_name = 'Bravo'") rows = await fav.list_favorites("disc1", "Varsoon") assert [r["character_name"] for r in rows] == ["Bravo", "Alpha"] diff --git a/tests/server/test_guild_recruitment.py b/tests/server/test_guild_recruitment.py index 251c29ad..a0e00950 100644 --- a/tests/server/test_guild_recruitment.py +++ b/tests/server/test_guild_recruitment.py @@ -13,8 +13,6 @@ import base64 import io import json -import sqlite3 -from pathlib import Path from unittest.mock import AsyncMock, patch import itsdangerous @@ -26,9 +24,8 @@ import backend.server.api.guild_recruitment as recruitment_api from backend.server import recruitment_sweep from backend.server.api.guild_recruitment import RECRUITMENT_TAGS, _process_logo -from backend.server.db import init_db from backend.server.db.guild_recruitment import store as gr -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn _TEST_SECRET = "pytest-session-secret-not-real-0123456789" @@ -36,16 +33,11 @@ _GID = 42 -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema @pytest.fixture(autouse=True) @@ -145,7 +137,7 @@ async def test_list_recruiting_is_world_scoped_ordered_and_blob_free(users_db): await _save_profile(world="Wuoshi", gid=3, name="Elsewhere") await _save_profile(gid=4, name="Hidden", recruiting=False) # Bump Alpha so it is the most recent edit. - with sqlite3.connect(users_db) as c: + with pg_conn(users_db) as c: c.execute("UPDATE guild_recruitment SET updated_at = updated_at + 60 WHERE guild_id = 1") rows = await gr.list_recruiting(_WORLD) assert [r["guild_name"] for r in rows] == ["Alpha", "Beta"] diff --git a/tests/server/test_item_watch_db.py b/tests/server/test_item_watch_db.py index ba6984ad..9de0f63c 100644 --- a/tests/server/test_item_watch_db.py +++ b/tests/server/test_item_watch_db.py @@ -1,8 +1,8 @@ """Per-server item-watch isolation tests — DB-helper layer (COV-005). Tests call DB helpers directly with explicit ``world`` args so they run -without a live HTTP layer and stay fast. The conftest plants a temp DB and -calls init_db() for us. +without a live HTTP layer and stay fast. Each test runs against an +isolated leased users schema (conftest ``users_schema``). HTTP-layer tests live in test_item_watch_routes.py. @@ -11,23 +11,26 @@ from __future__ import annotations -import sqlite3 - import pytest -from backend.server import db from backend.server.db import ( add_item_watch, list_item_watches, remove_item_watch, upsert_user, ) +from tests.fixtures.pg import pg_conn # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -_PATH = db.DB_PATH # redirected to pytest tmpdir by conftest.py + +@pytest.fixture(autouse=True) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema async def _seed_user(discord_id: str) -> None: @@ -45,30 +48,36 @@ async def _seed_user(discord_id: str) -> None: # --------------------------------------------------------------------------- -def test_world_column_exists_in_schema(): - """init_db (run by conftest) must have added the world column to item_watch.""" - with sqlite3.connect(_PATH) as conn: - cols = {row[1] for row in conn.execute("PRAGMA table_info(item_watch)")} +def test_world_column_exists_in_schema(users_db): + """The migration schema must carry the world column on item_watch.""" + with pg_conn(users_db) as conn: + rows = conn.execute( + "SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = 'item_watch'", + (users_db,), + ).fetchall() + cols = {r["column_name"] for r in rows} assert "world" in cols, "world column not found in item_watch" -def test_unique_constraint_includes_world(): +def test_unique_constraint_includes_world(users_db): """The UNIQUE constraint on item_watch must cover (world, guild_name, character_name, item_id). - We verify this by checking the sqlite_master index definition — the old - constraint was (guild_name, character_name, item_id) only. + We verify this against pg_index (which covers both UNIQUE constraints and + unique indexes) — the old SQLite constraint was (guild_name, + character_name, item_id) only. """ - with sqlite3.connect(_PATH) as conn: - # The inline UNIQUE on the CREATE TABLE becomes an unnamed auto-index - # in sqlite_master. We can inspect the table's info via index_list. - index_rows = list(conn.execute("PRAGMA index_list(item_watch)")) - # Collect all column sets for unique indexes - unique_col_sets: list[frozenset[str]] = [] - for row in index_rows: - if row[2] == 1: # unique flag - idx_name = row[1] - cols = frozenset(r[2] for r in conn.execute(f"PRAGMA index_info({idx_name!r})")) - unique_col_sets.append(cols) + with pg_conn(users_db) as conn: + # search_path is the leased schema, so the regclass resolves there. + rows = conn.execute( + """ + SELECT i.indexrelid::regclass::text AS idx_name, array_agg(a.attname) AS cols + FROM pg_index i + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey) + WHERE i.indrelid = 'item_watch'::regclass AND i.indisunique + GROUP BY i.indexrelid + """ + ).fetchall() + unique_col_sets = [frozenset(r["cols"]) for r in rows] target = frozenset({"world", "guild_name", "character_name", "item_id"}) assert target in unique_col_sets, ( diff --git a/tests/server/test_raid_live.py b/tests/server/test_raid_live.py index 01d159b8..71d28594 100644 --- a/tests/server/test_raid_live.py +++ b/tests/server/test_raid_live.py @@ -40,7 +40,7 @@ def mc(wd, mn): # Tue 22:00-01:00 def test_team_scheduled_now_uses_team_timezone(): - team = {"primary_tz": "America/New_York", "raids": [{"days": "2,4", "start_min": 1200, "end_min": 1380}]} + team = {"primary_tz": "America/New_York", "raids": [{"days": [2, 4], "start_min": 1200, "end_min": 1380}]} # Wed 02:00 UTC (winter) == Tue 21:00 EST → inside the Tue window. assert rl.team_scheduled_now(team, now=datetime(2026, 1, 14, 2, 0, tzinfo=UTC)) is True # Wed 18:00 UTC == Wed 13:00 EST → not a raid day/time. @@ -90,7 +90,7 @@ async def test_refresh_populates_cache_for_scheduled_live_teams(monkeypatch): "name": "Main", "primary_tz": "America/New_York", "twitch_login": "foo", - "raids": [{"days": "2,4", "start_min": 1200, "end_min": 1380}], + "raids": [{"days": [2, 4], "start_min": 1200, "end_min": 1380}], }, # Not scheduled now (Mondays only) → excluded even if it were live. { @@ -99,7 +99,7 @@ async def test_refresh_populates_cache_for_scheduled_live_teams(monkeypatch): "name": "T", "primary_tz": "America/New_York", "twitch_login": "bar", - "raids": [{"days": "1", "start_min": 1200, "end_min": 1380}], + "raids": [{"days": [1], "start_min": 1200, "end_min": 1380}], }, ] with ( diff --git a/tests/server/test_raid_planning.py b/tests/server/test_raid_planning.py index e1c755c5..73f9ce21 100644 --- a/tests/server/test_raid_planning.py +++ b/tests/server/test_raid_planning.py @@ -1,43 +1,50 @@ """Raid-planning DB layer + API tests. -DB layer runs against a temp users.db (stores re-pointed by the autouse -fixture). The API is tested with the ``app`` fixture, a signed session -cookie for auth, and mocked ``_officer_chars`` / ``_roster_rank_map`` / -roster helpers (same pattern as test_raid_schedule.py). +DB layer runs against a leased scratch Postgres schema (stores re-pointed +by the autouse fixture). The API is tested with the ``app`` fixture, a +signed session cookie for auth, and mocked ``_officer_chars`` / +``_roster_rank_map`` / roster helpers (same pattern as +test_raid_schedule.py). """ from __future__ import annotations import datetime as dt -from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest from httpx import ASGITransport, AsyncClient -from backend.server.db import init_db -from tests.fixtures.users_db import point_users_db_at +from backend.server.db.availability import store as availability_db +from backend.server.db.raid_planning import store as planning_db +from tests.fixtures.pg import pg_conn # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db +@pytest.fixture(autouse=True) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) - +def _seed_common_users(users_db: str) -> None: + """user_availability.discord_id and character_claims.discord_id are + enforced FKs to users on Postgres (SQLite never enabled the pragma) — + seed the ids the tests write with.""" + with pg_conn(users_db) as conn: + for did in ("u1", "u2", "member-1"): + conn.execute( + "INSERT INTO users (discord_id, discord_name, access_status) " + "VALUES (%s, %s, 'approved') ON CONFLICT (discord_id) DO NOTHING", + (did, did), + ) -from backend.server.db.availability import store as availability_db # noqa: E402 -from backend.server.db.raid_planning import store as planning_db # noqa: E402 _WORLD = "Varsoon" _GUILD = "Exordium" @@ -667,14 +674,12 @@ def test_merge_availability_newest_wins(): @pytest.mark.asyncio -async def test_planner_officer_edit_overrides_stale_self_declaration(app): +async def test_planner_officer_edit_overrides_stale_self_declaration(app, users_db): """The live complaint: a player self-declared AFK, the officer changes the character to Tentative on the planner — the newer officer edit must actually show (the old always-player-wins merge silently masked it), and an officer 'Available' must clear the badge, until the player re-declares (newest edit wins again).""" - import sqlite3 as _sq - from backend.server.db import upsert_user from backend.server.db.claims import store as claims_db @@ -686,9 +691,8 @@ async def test_planner_officer_edit_overrides_stale_self_declaration(app): # The player declared AFK a while ago (age the stamp). await availability_db.set_days("member-1", {today: "afk"}) - with _sq.connect(availability_db.path) as conn: + with pg_conn(users_db) as conn: conn.execute("UPDATE user_availability SET updated_at = 1000 WHERE discord_id = 'member-1'") - conn.commit() # Officer corrects the character to tentative (stamped now → newer). await availability_db.set_character_days(_WORLD, "Tanky", {today: "tentative"}, set_by="officer-1") diff --git a/tests/server/test_raid_schedule.py b/tests/server/test_raid_schedule.py index 873fcf9d..51ede89c 100644 --- a/tests/server/test_raid_schedule.py +++ b/tests/server/test_raid_schedule.py @@ -1,7 +1,7 @@ """Raid-schedule DB layer + API tests. -DB layer is exercised against a temp users.db (stores re-pointed by the -autouse fixture). The API is +DB layer is exercised against a leased scratch Postgres schema (stores +re-pointed by the autouse fixture). The API is tested with the ``app`` fixture: a signed session cookie for auth + mocked ``_officer_chars`` / db helpers (same pattern as test_item_watch_routes.py). """ @@ -10,7 +10,6 @@ import base64 import json -from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import itsdangerous @@ -18,9 +17,8 @@ from better_profanity import profanity from httpx import ASGITransport, AsyncClient -from backend.server.db import init_db from backend.server.db.raid_schedule import store as rs -from tests.fixtures.users_db import point_users_db_at +from tests.fixtures.pg import pg_conn _TEST_SECRET = "pytest-session-secret-not-real-0123456789" @@ -37,30 +35,36 @@ def sentinel_profanity(): # --------------------------------------------------------------------------- -# DB layer (temp users.db) +# DB layer (leased scratch schema) # --------------------------------------------------------------------------- -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) # creates raid_teams / raid_slots (+ asserts schema) - return db +@pytest.fixture(autouse=True) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" + return users_schema -@pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - """Point users.db (constant + every domain store) at this test's temp DB.""" - point_users_db_at(monkeypatch, users_db) +def _ensure_user(schema: str, discord_id: str) -> None: + """raid_teams.updated_by is an enforced FK to users on Postgres — + seed the id the DB-layer tests write with.""" + with pg_conn(schema) as conn: + conn.execute( + "INSERT INTO users (discord_id, discord_name, access_status) " + "VALUES (%s, %s, 'approved') ON CONFLICT (discord_id) DO NOTHING", + (discord_id, discord_id), + ) async def test_replace_then_get_round_trips(users_db): + _ensure_user(users_db, "disc1") teams = [ { "name": "Team 1", "primary_tz": "America/New_York", "twitch_login": "foochan", - "raids": [{"days": "2,4", "start_min": 1200, "end_min": 1380, "label": "Prog"}], + "raids": [{"days": [2, 4], "start_min": 1200, "end_min": 1380, "label": "Prog"}], } ] await rs.replace_schedule("Varsoon", "Exordium", teams, "disc1") @@ -68,11 +72,12 @@ async def test_replace_then_get_round_trips(users_db): assert len(got) == 1 assert got[0]["name"] == "Team 1" assert got[0]["twitch_login"] == "foochan" - assert got[0]["raids"][0]["days"] == "2,4" + assert got[0]["raids"][0]["days"] == [2, 4] assert got[0]["raids"][0]["start_min"] == 1200 async def test_replace_is_a_full_replace_and_scoped(users_db): + _ensure_user(users_db, "disc1") await rs.replace_schedule( "Varsoon", "Exordium", @@ -181,7 +186,7 @@ async def test_put_officer_saves_and_converts(app): team = saved_teams[0] assert team["twitch_login"] == "mychannel" # url → login assert team["raids"][0]["start_min"] == 1200 # "20:00" → minutes - assert team["raids"][0]["days"] == "2,4" + assert team["raids"][0]["days"] == [2, 4] async def test_put_rejects_too_many_teams(app): diff --git a/tests/server/test_rankings.py b/tests/server/test_rankings.py index ee254ee5..6c387f31 100644 --- a/tests/server/test_rankings.py +++ b/tests/server/test_rankings.py @@ -1,7 +1,6 @@ from __future__ import annotations from backend.server.api.rankings import _apply_percentiles, _scope_for -from backend.server.db.servers import ServersStore class TestApplyPercentiles: @@ -447,7 +446,6 @@ def test_resolve_boss_uses_zones_db_for_raids(self): from backend.server.parses import db as pdb from backend.server.parses.models import Combatant, CombatantSnapshot, Encounter -from tests.fixtures.users_db import point_users_db_at def _ins(conn, encid, title, *, success, players, guild, duration): @@ -640,18 +638,14 @@ async def test_rankings_rejects_bad_size(app, rankings_db): @pytest.mark.asyncio -async def test_rankings_default_xpac_per_server(app, monkeypatch, tmp_path): +async def test_rankings_default_xpac_per_server(app, users_schema): """The rankings /filters endpoint's default_expansion reflects the active server's current_xpac, not a global env var.""" from backend.server import db, server_context - # Point the DB at a temp file and seed a Wuoshi server row. - p = tmp_path / "users.db" - db.init_db(p) - point_users_db_at(monkeypatch, p) - ServersStore(p).upsert_server_settings_sync( - "Wuoshi", max_level=70, current_xpac="Echoes of Faydwer", launch_dt=None - ) + # Isolated leased schema (seeded with Varsoon + Wuoshi rows) — set + # Wuoshi's xpac and reload the registry from it. + db.upsert_server_settings_sync("Wuoshi", max_level=70, current_xpac="Echoes of Faydwer", launch_dt=None) server_context.load_registry() raid_tree = [ @@ -673,7 +667,7 @@ async def test_rankings_default_xpac_per_server(app, monkeypatch, tmp_path): @pytest.mark.asyncio -async def test_rankings_leaderboard_is_world_scoped(app, monkeypatch, tmp_path): +async def test_rankings_leaderboard_is_world_scoped(app, monkeypatch, tmp_path, users_schema): """The /rankings leaderboard endpoint must return per-world data via the executor call path. @@ -688,12 +682,10 @@ async def test_rankings_leaderboard_is_world_scoped(app, monkeypatch, tmp_path): from backend.server import db, server_context from backend.server.api import rankings as rk - # Register both servers so x-server: wuoshi resolves correctly. - p = tmp_path / "users.db" - db.init_db(p) - point_users_db_at(monkeypatch, p) - ServersStore(p).upsert_server_settings_sync("Varsoon", max_level=50, current_xpac=None, launch_dt=None) - ServersStore(p).upsert_server_settings_sync("Wuoshi", max_level=70, current_xpac=None, launch_dt=None) + # Register both servers so x-server: wuoshi resolves correctly (the + # leased schema seeds both rows; refresh their settings and reload). + db.upsert_server_settings_sync("Varsoon", max_level=50, current_xpac=None, launch_dt=None) + db.upsert_server_settings_sync("Wuoshi", max_level=70, current_xpac=None, launch_dt=None) server_context.load_registry() # Seed a Varsoon boss kill and a distinct Wuoshi boss kill. diff --git a/tests/server/test_role_requests.py b/tests/server/test_role_requests.py index 54868d48..4637a05d 100644 --- a/tests/server/test_role_requests.py +++ b/tests/server/test_role_requests.py @@ -3,9 +3,9 @@ from __future__ import annotations -import sqlite3 from unittest.mock import AsyncMock, patch +import psycopg.errors import pytest from httpx import ASGITransport, AsyncClient @@ -90,7 +90,7 @@ async def test_submit_role_request_409_when_pending_exists(app): patch("backend.server.api.role_requests.has_role", return_value=False), patch( "backend.server.api.role_requests.create_role_request", - side_effect=sqlite3.IntegrityError("UNIQUE constraint failed"), + side_effect=psycopg.errors.UniqueViolation("duplicate key value violates unique constraint"), ), ): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: diff --git a/tests/server/test_server_context.py b/tests/server/test_server_context.py index ef151473..f73f61e6 100644 --- a/tests/server/test_server_context.py +++ b/tests/server/test_server_context.py @@ -1,38 +1,35 @@ from __future__ import annotations -from backend.server import server_context as sc +import pytest +from httpx import ASGITransport, AsyncClient +from backend.server import server_context as sc -def _seed(monkeypatch, tmp_path): - from backend.server import db - p = tmp_path / "users.db" - db.init_db(p) - point_users_db_at(monkeypatch, p) +@pytest.fixture(autouse=True) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``) with the + registry loaded from its seeded servers rows (Varsoon default + Wuoshi).""" sc.load_registry() - return p + return users_schema -def test_resolve_known_subdomain(monkeypatch, tmp_path): - _seed(monkeypatch, tmp_path) +def test_resolve_known_subdomain(): assert sc.resolve_host("wuoshi.eq2lexicon.com").world == "Wuoshi" assert sc.resolve_host("varsoon.eq2lexicon.com").world == "Varsoon" -def test_resolve_unknown_falls_back_to_default(monkeypatch, tmp_path): - _seed(monkeypatch, tmp_path) +def test_resolve_unknown_falls_back_to_default(): assert sc.resolve_host("localhost:8000").world == "Varsoon" assert sc.resolve_host("eq2lexicon.com").world == "Varsoon" assert sc.resolve_host("").world == "Varsoon" -def test_current_world_default_outside_request(monkeypatch, tmp_path): - _seed(monkeypatch, tmp_path) +def test_current_world_default_outside_request(): assert sc.current_world() == "Varsoon" -def test_contextvar_roundtrip(monkeypatch, tmp_path): - _seed(monkeypatch, tmp_path) +def test_contextvar_roundtrip(): wuoshi = sc.resolve_host("wuoshi.eq2lexicon.com") token = sc.set_active_server(wuoshi) try: @@ -43,8 +40,7 @@ def test_contextvar_roundtrip(monkeypatch, tmp_path): assert sc.current_world() == "Varsoon" -def test_override_ignored_when_disabled(monkeypatch, tmp_path): - _seed(monkeypatch, tmp_path) +def test_override_ignored_when_disabled(monkeypatch): # Simulate production: the X-Server/?server= override must be ignored. monkeypatch.setattr(sc, "_ALLOW_OVERRIDE", False) assert sc.resolve_host("varsoon.eq2lexicon.com", override="wuoshi").world == "Varsoon" @@ -53,14 +49,11 @@ def test_override_ignored_when_disabled(monkeypatch, tmp_path): assert sc.resolve_host("varsoon.eq2lexicon.com", override="wuoshi").world == "Wuoshi" -def test_default_server_returns_is_default_server(monkeypatch, tmp_path): +def test_default_server_returns_is_default_server(): """default_server() should return whichever server has is_default=True, even if it doesn't match EQ2_WORLD.""" from backend.server import db - p = tmp_path / "users.db" - db.init_db(p) - point_users_db_at(monkeypatch, p) # Set Wuoshi as the default in the DB. db.set_default_server_sync("Wuoshi") @@ -69,19 +62,12 @@ def test_default_server_returns_is_default_server(monkeypatch, tmp_path): assert sc.default_server().world == "Wuoshi" -import pytest -from httpx import ASGITransport, AsyncClient - -from tests.fixtures.users_db import point_users_db_at - - @pytest.mark.asyncio -async def test_middleware_sets_world_from_host(monkeypatch, tmp_path): +async def test_middleware_sets_world_from_host(): from fastapi import FastAPI from backend.server import server_context as sc2 - _seed(monkeypatch, tmp_path) app = FastAPI() app.add_middleware(sc2.ServerContextMiddleware) diff --git a/tests/server/test_server_route.py b/tests/server/test_server_route.py index a4834582..96d0d8cd 100644 --- a/tests/server/test_server_route.py +++ b/tests/server/test_server_route.py @@ -3,17 +3,19 @@ import pytest from httpx import ASGITransport, AsyncClient -from tests.fixtures.users_db import point_users_db_at +@pytest.fixture(autouse=True) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``) with the + registry loaded from its seeded servers rows (Varsoon default + Wuoshi).""" + from backend.server import server_context -@pytest.mark.asyncio -async def test_server_endpoint_reflects_subdomain(app, monkeypatch, tmp_path): - from backend.server import db, server_context - - p = tmp_path / "users.db" - db.init_db(p) - point_users_db_at(monkeypatch, p) server_context.load_registry() + return users_schema + + +@pytest.mark.asyncio +async def test_server_endpoint_reflects_subdomain(app): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: r = await c.get("/api/server", headers={"host": "wuoshi.eq2lexicon.com"}) assert r.status_code == 200 @@ -25,13 +27,7 @@ async def test_server_endpoint_reflects_subdomain(app, monkeypatch, tmp_path): @pytest.mark.asyncio -async def test_server_endpoint_unknown_host_defaults(app, monkeypatch, tmp_path): - from backend.server import db, server_context - - p = tmp_path / "users.db" - db.init_db(p) - point_users_db_at(monkeypatch, p) - server_context.load_registry() +async def test_server_endpoint_unknown_host_defaults(app): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: r = await c.get("/api/server", headers={"host": "localhost"}) assert r.json()["world"] == "Varsoon" diff --git a/tests/server/test_servers_db.py b/tests/server/test_servers_db.py index 3c22adcb..9c4108b0 100644 --- a/tests/server/test_servers_db.py +++ b/tests/server/test_servers_db.py @@ -1,215 +1,144 @@ -from __future__ import annotations +"""servers registry store (sync) + schema invariants. + +Converted from the SQLite init_db/migration era: the leased scratch schema +(conftest ``users_schema``) is built from db/migrations/0001_users.sql, whose +``-- seeds`` section inserts the Varsoon (default) + Wuoshi rows. The old +legacy-ALTER migration regressions survive here as assertions that the +Postgres schema carries the per-server columns/indexes outright, and the +"second init_db is a no-op" guarantee becomes "re-running the idempotent +seeds never clobbers an admin edit". +""" -import sqlite3 +from __future__ import annotations -from backend.server import db from backend.server.db.servers import ServersStore +from tests.fixtures.pg import leaser, pg_conn + +def _rerun_seeds(schema: str) -> None: + """Re-apply the migration's idempotent ``-- seeds`` section — the redeploy + analog of the old "second init_db" call.""" + with pg_conn(schema) as conn: + conn.execute(leaser._seeds_sql()) -def test_servers_seeded_and_lookups(tmp_path): - p = tmp_path / "users.db" - db.init_db(p) - rows = ServersStore(p).list_servers_sync() + +def test_servers_seeded_and_lookups(users_schema): + rows = ServersStore(users_schema).list_servers_sync() worlds = {r["world"] for r in rows} assert {"Varsoon", "Wuoshi"} <= worlds - v = ServersStore(p).get_server_by_subdomain_sync("varsoon") + v = ServersStore(users_schema).get_server_by_subdomain_sync("varsoon") assert v is not None and v["world"] == "Varsoon" - w = ServersStore(p).get_server_by_world_sync("Wuoshi") + w = ServersStore(users_schema).get_server_by_world_sync("Wuoshi") assert w is not None and w["subdomain"] == "wuoshi" - assert ServersStore(p).get_server_by_subdomain_sync("nope") is None + assert ServersStore(users_schema).get_server_by_subdomain_sync("nope") is None -def test_upsert_server_updates_settings(tmp_path): - p = tmp_path / "users.db" - db.init_db(p) - ServersStore(p).upsert_server_settings_sync( - "Wuoshi", max_level=70, current_xpac="Sentinel's Fate", launch_dt="2026-07-01T18:00:00Z" +def test_upsert_server_updates_settings(users_schema): + ServersStore(users_schema).upsert_server_settings_sync( + "Wuoshi", max_level=90, current_xpac="Sentinel's Fate", launch_dt="2026-07-01T18:00:00Z" ) - w = ServersStore(p).get_server_by_world_sync("Wuoshi") - assert w["max_level"] == 70 + w = ServersStore(users_schema).get_server_by_world_sync("Wuoshi") + assert w["max_level"] == 90 assert w["current_xpac"] == "Sentinel's Fate" assert w["launch_dt"] == "2026-07-01T18:00:00Z" -def test_second_init_db_preserves_upserted_settings(tmp_path): - p = tmp_path / "users.db" - db.init_db(p) - ServersStore(p).upsert_server_settings_sync("Wuoshi", max_level=70, current_xpac="Sentinel's Fate", launch_dt=None) - # A second init_db (e.g. container restart) must not reset the admin edit. - db.init_db(p) - w = ServersStore(p).get_server_by_world_sync("Wuoshi") - assert w["max_level"] == 70 +def test_seed_rerun_preserves_upserted_settings(users_schema): + ServersStore(users_schema).upsert_server_settings_sync( + "Wuoshi", max_level=90, current_xpac="Sentinel's Fate", launch_dt=None + ) + # A seeds re-run (e.g. redeploy re-applying the migration's idempotent + # seeds) must not reset the admin edit. + _rerun_seeds(users_schema) + w = ServersStore(users_schema).get_server_by_world_sync("Wuoshi") + assert w["max_level"] == 90 assert w["current_xpac"] == "Sentinel's Fate" -def test_init_db_migrates_legacy_db_without_world_columns(tmp_path): - """Regression: init_db must not crash on a pre-per-server users.db whose - character_claims / item_watch lack the `world` column. - - The bug: the idx_claims_world index lived in _SCHEMA (run by executescript - BEFORE the ALTER that adds `world`), so on an existing DB it raised - 'no such column: world' and crashed app startup -> whole site down. +def test_schema_carries_per_server_columns_and_indexes(users_schema): + """Descendant of the SQLite legacy-migration regression (init_db crashing + on a pre-per-server users.db): on Postgres the per-server columns and the + world index are part of the schema outright — assert they exist. """ - import sqlite3 - - p = tmp_path / "users.db" - conn = sqlite3.connect(p) - conn.executescript( - """ - CREATE TABLE users ( - discord_id TEXT PRIMARY KEY, discord_name TEXT NOT NULL, - discord_username TEXT, avatar TEXT, - first_seen INTEGER NOT NULL DEFAULT (strftime('%s','now')), - last_seen INTEGER NOT NULL DEFAULT (strftime('%s','now')), - access_status TEXT NOT NULL DEFAULT 'pending' - ); - CREATE TABLE character_claims ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - discord_id TEXT NOT NULL, character_name TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - requested_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - reviewed_at INTEGER, reviewed_by TEXT, note TEXT - ); - CREATE TABLE item_watch ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - guild_name TEXT NOT NULL, character_name TEXT NOT NULL, - item_id INTEGER NOT NULL, item_name TEXT NOT NULL, - added_by TEXT NOT NULL, added_by_name TEXT NOT NULL, - added_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - first_seen_at INTEGER, last_seen_at INTEGER, last_checked_at INTEGER, - UNIQUE(guild_name, character_name, item_id) - ); - INSERT INTO users (discord_id, discord_name) VALUES ('u1', 'U1'); - INSERT INTO character_claims (discord_id, character_name, status) - VALUES ('u1', 'Sihtric', 'approved'); - INSERT INTO item_watch (guild_name, character_name, item_id, item_name, added_by, added_by_name) - VALUES ('Exordium', 'Sihtric', 42, 'Sword', 'u1', 'U1'); - """ - ) - conn.commit() - conn.close() - - # Must NOT raise (this is the exact crash that took the site down). - db.init_db(p) - - conn = sqlite3.connect(p) - try: - claims_cols = {r[1] for r in conn.execute("PRAGMA table_info(character_claims)")} - assert "world" in claims_cols and "is_primary" in claims_cols - watch_cols = {r[1] for r in conn.execute("PRAGMA table_info(item_watch)")} - assert "world" in watch_cols - # Legacy rows backfilled to Varsoon. - assert ( - conn.execute("SELECT world FROM character_claims WHERE character_name='Sihtric'").fetchone()[0] == "Varsoon" - ) - assert conn.execute("SELECT world FROM item_watch WHERE item_id=42").fetchone()[0] == "Varsoon" - # The world index now exists. - idx = {r[1] for r in conn.execute("PRAGMA index_list(character_claims)")} - assert "idx_claims_world" in idx - finally: - conn.close() - - # Idempotent: a second boot on the migrated DB is a clean no-op. - db.init_db(p) + with pg_conn(users_schema) as conn: + claims_cols = { + r["column_name"] + for r in conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'character_claims'", + (users_schema,), + ).fetchall() + } + watch_cols = { + r["column_name"] + for r in conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'item_watch'", + (users_schema,), + ).fetchall() + } + idx = { + r["indexname"] + for r in conn.execute( + "SELECT indexname FROM pg_indexes WHERE schemaname = %s AND tablename = 'character_claims'", + (users_schema,), + ).fetchall() + } + assert "world" in claims_cols and "is_primary" in claims_cols + assert "world" in watch_cols + assert "idx_claims_world" in idx # --------------------------------------------------------------------------- -# is_default migration tests +# is_default invariants # --------------------------------------------------------------------------- -def test_init_db_adds_is_default_to_legacy_servers_table(monkeypatch, tmp_path): - """Regression: init_db must not crash against a prod servers table that has NO - is_default column, must add the column, and must ensure exactly one row has - is_default=1 afterwards. A second init_db call must be idempotent. +def test_exactly_one_default_server_seeded(users_schema): + """Descendant of the SQLite is_default migration regression: the seeds + must leave exactly one default row (Varsoon), and a seeds re-run must be + idempotent — still exactly one default. """ - import os - - # Simulate the production DB shape: servers table WITHOUT is_default. - p = tmp_path / "legacy.db" - conn = sqlite3.connect(p) - conn.executescript( - """ - CREATE TABLE servers ( - world TEXT PRIMARY KEY, - subdomain TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - max_level INTEGER NOT NULL, - current_xpac TEXT, - launch_dt TEXT, - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) - ); - INSERT INTO servers (world, subdomain, display_name, max_level) - VALUES ('Varsoon', 'varsoon', 'Varsoon', 50); - INSERT INTO servers (world, subdomain, display_name, max_level) - VALUES ('Wuoshi', 'wuoshi', 'Wuoshi', 50); - """ - ) - conn.commit() - conn.close() - - # Ensure EQ2_WORLD resolves to Varsoon (the default env). - monkeypatch.setenv("EQ2_WORLD", "Varsoon") - - # Must NOT raise. - db.init_db(p) - - # Column now exists and exactly one row has is_default=1. - conn = sqlite3.connect(p) - try: - cols = {r[1] for r in conn.execute("PRAGMA table_info(servers)")} - assert "is_default" in cols, "is_default column must be added by migration" + with pg_conn(users_schema) as conn: + rows = conn.execute("SELECT world FROM servers WHERE is_default = 1").fetchall() + assert [r["world"] for r in rows] == ["Varsoon"] - defaults = conn.execute("SELECT world FROM servers WHERE is_default = 1").fetchall() - assert len(defaults) == 1, f"Expected exactly 1 default, got {defaults}" - assert defaults[0][0] == "Varsoon", "EQ2_WORLD server should be the default" - finally: - conn.close() + _rerun_seeds(users_schema) + with pg_conn(users_schema) as conn: + rows = conn.execute("SELECT world FROM servers WHERE is_default = 1").fetchall() + assert len(rows) == 1, f"After seed re-run, expected 1 default, got {rows}" - # Second init_db must be idempotent — still exactly one default. - db.init_db(p) - conn = sqlite3.connect(p) - try: - defaults = conn.execute("SELECT world FROM servers WHERE is_default = 1").fetchall() - assert len(defaults) == 1, f"After second init_db, expected 1 default, got {defaults}" - finally: - conn.close() - - -def test_set_default_server_clears_others(tmp_path): +def test_set_default_server_clears_others(users_schema): """set_default_server_sync atomically swaps the default between servers.""" - p = tmp_path / "users.db" - db.init_db(p) - - # After init the default should be Varsoon (EQ2_WORLD default). - rows = ServersStore(p).list_servers_sync() + # After seeding the default should be Varsoon. + rows = ServersStore(users_schema).list_servers_sync() defaults = [r for r in rows if r["is_default"]] assert len(defaults) == 1 # Set Wuoshi as default — Varsoon must flip to 0. - result = ServersStore(p).set_default_server_sync("Wuoshi") + result = ServersStore(users_schema).set_default_server_sync("Wuoshi") assert result is True - rows = ServersStore(p).list_servers_sync() + rows = ServersStore(users_schema).list_servers_sync() wuoshi = next(r for r in rows if r["world"] == "Wuoshi") varsoon = next(r for r in rows if r["world"] == "Varsoon") assert wuoshi["is_default"] is True assert varsoon["is_default"] is False # Flip back to Varsoon. - result = ServersStore(p).set_default_server_sync("Varsoon") + result = ServersStore(users_schema).set_default_server_sync("Varsoon") assert result is True - rows = ServersStore(p).list_servers_sync() + rows = ServersStore(users_schema).list_servers_sync() wuoshi = next(r for r in rows if r["world"] == "Wuoshi") varsoon = next(r for r in rows if r["world"] == "Varsoon") assert varsoon["is_default"] is True assert wuoshi["is_default"] is False # Unknown world: returns False and does NOT leave zero defaults. - result = ServersStore(p).set_default_server_sync("Nope") + result = ServersStore(users_schema).set_default_server_sync("Nope") assert result is False - rows = ServersStore(p).list_servers_sync() + rows = ServersStore(users_schema).list_servers_sync() defaults = [r for r in rows if r["is_default"]] assert len(defaults) == 1, "Must always have exactly one default even after failed set" diff --git a/tests/server/test_site_settings.py b/tests/server/test_site_settings.py index 932bc949..351eeb61 100644 --- a/tests/server/test_site_settings.py +++ b/tests/server/test_site_settings.py @@ -3,33 +3,25 @@ from __future__ import annotations -from pathlib import Path from unittest.mock import patch import pytest from httpx import ASGITransport, AsyncClient from backend.server import server_context -from backend.server.db import init_db from backend.server.db.site_settings import DISCORD_INVITE_URL_KEY, SiteSettingsStore from backend.server.db.site_settings import store as site_settings from tests.fixtures.users import make_fake_admin -from tests.fixtures.users_db import point_users_db_at _ADMIN = make_fake_admin(id="admin-1", username="boss") -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db - - @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``.""" server_context.load_registry() + return users_schema # --------------------------------------------------------------------------- diff --git a/tests/server/test_users_db_roles.py b/tests/server/test_users_db_roles.py index 3f00ebe4..644626b1 100644 --- a/tests/server/test_users_db_roles.py +++ b/tests/server/test_users_db_roles.py @@ -1,6 +1,6 @@ """Tests for web.db.users role + role_request helpers — COV-012. -Uses a per-test temp DB via the project conftest _TEST_DB_DIR pattern. +Runs against an isolated leased users schema (conftest ``users_schema``). Covers: grant_role, revoke_role, list_roles_for_user, has_role, create_role_request, list_role_requests, review_and_grant_role, withdraw_role_request, user_has_capability_via_db, role_has_capability, @@ -11,9 +11,6 @@ from __future__ import annotations -import tempfile -from pathlib import Path - import pytest from backend.server.db import ( @@ -23,7 +20,6 @@ get_user_access_status, grant_role, has_role, - init_db, list_all_users, list_pending_users, list_role_assignments, @@ -38,23 +34,17 @@ user_has_capability_via_db, withdraw_role_request, ) -from tests.fixtures.users_db import point_users_db_at - - -@pytest.fixture -def db_path(tmp_path: Path) -> Path: - p = tmp_path / "users.db" - init_db(p) - return p +from tests.fixtures.pg import pg_conn @pytest.fixture(autouse=True) -def _stores_at_db_path(db_path: Path, monkeypatch: pytest.MonkeyPatch): - """Point users.db (constant + every domain store) at this test's temp DB.""" - point_users_db_at(monkeypatch, db_path) +def db_path(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``db_path``.""" + return users_schema -async def _seed_user(db_path: Path, discord_id: str = "user-1", name: str = "TestUser") -> None: +async def _seed_user(db_path: str, discord_id: str = "user-1", name: str = "TestUser") -> None: await upsert_user(discord_id, name, name.lower(), None) @@ -65,44 +55,44 @@ async def _seed_user(db_path: Path, discord_id: str = "user-1", name: str = "Tes class TestRoleHelpers: @pytest.mark.asyncio - async def test_grant_role_returns_true_on_insert(self, db_path: Path): + async def test_grant_role_returns_true_on_insert(self, db_path: str): await _seed_user(db_path) result = await grant_role("user-1", "contributor", granted_by="admin-1") assert result is True @pytest.mark.asyncio - async def test_grant_role_idempotent_returns_false(self, db_path: Path): + async def test_grant_role_idempotent_returns_false(self, db_path: str): await _seed_user(db_path) await grant_role("user-1", "contributor", granted_by="admin-1") second = await grant_role("user-1", "contributor", granted_by="admin-1") assert second is False @pytest.mark.asyncio - async def test_revoke_role_returns_true_when_held(self, db_path: Path): + async def test_revoke_role_returns_true_when_held(self, db_path: str): await _seed_user(db_path) await grant_role("user-1", "contributor", granted_by="admin-1") assert await revoke_role("user-1", "contributor") is True @pytest.mark.asyncio - async def test_revoke_role_returns_false_when_not_held(self, db_path: Path): + async def test_revoke_role_returns_false_when_not_held(self, db_path: str): await _seed_user(db_path) assert await revoke_role("user-1", "contributor") is False @pytest.mark.asyncio - async def test_list_roles_for_user(self, db_path: Path): + async def test_list_roles_for_user(self, db_path: str): await _seed_user(db_path) await grant_role("user-1", "contributor", granted_by="admin-1") roles = await list_roles_for_user("user-1") assert "contributor" in roles @pytest.mark.asyncio - async def test_has_role_returns_true_when_granted(self, db_path: Path): + async def test_has_role_returns_true_when_granted(self, db_path: str): await _seed_user(db_path) await grant_role("user-1", "contributor", granted_by="admin-1") assert await has_role("user-1", "contributor") is True @pytest.mark.asyncio - async def test_has_role_returns_false_when_absent(self, db_path: Path): + async def test_has_role_returns_false_when_absent(self, db_path: str): await _seed_user(db_path) assert await has_role("user-1", "contributor") is False @@ -114,7 +104,7 @@ async def test_has_role_returns_false_when_absent(self, db_path: Path): class TestRoleRequestHelpers: @pytest.mark.asyncio - async def test_create_role_request_returns_int_id(self, db_path: Path): + async def test_create_role_request_returns_int_id(self, db_path: str): await _seed_user(db_path) request_id = await create_role_request("user-1", "contributor", user_note="please") assert isinstance(request_id, int) @@ -124,7 +114,7 @@ async def test_create_role_request_returns_int_id(self, db_path: Path): assert row["role"] == "contributor" @pytest.mark.asyncio - async def test_list_role_requests_pending_oldest_first(self, db_path: Path): + async def test_list_role_requests_pending_oldest_first(self, db_path: str): await _seed_user(db_path, discord_id="user-1") await _seed_user(db_path, discord_id="user-2", name="Other") await create_role_request("user-1", "contributor", user_note=None) @@ -133,7 +123,7 @@ async def test_list_role_requests_pending_oldest_first(self, db_path: Path): assert rows[0]["id"] <= rows[-1]["id"] # oldest (lower id) first @pytest.mark.asyncio - async def test_list_role_requests_approved_newest_first(self, db_path: Path): + async def test_list_role_requests_approved_newest_first(self, db_path: str): await _seed_user(db_path, discord_id="user-1") await _seed_user(db_path, discord_id="user-2", name="Other") r1_id = await create_role_request("user-1", "contributor", user_note=None) @@ -152,7 +142,7 @@ async def test_list_role_requests_approved_newest_first(self, db_path: Path): class TestReviewAndGrantRole: @pytest.mark.asyncio - async def test_atomic_approve_and_grant(self, db_path: Path): + async def test_atomic_approve_and_grant(self, db_path: str): await _seed_user(db_path) rr_id = await create_role_request("user-1", "contributor", user_note=None) result = await review_and_grant_role(rr_id, "approved", "admin-1") @@ -162,7 +152,7 @@ async def test_atomic_approve_and_grant(self, db_path: Path): assert await has_role("user-1", "contributor") @pytest.mark.asyncio - async def test_idempotent_user_already_has_role(self, db_path: Path): + async def test_idempotent_user_already_has_role(self, db_path: str): """If user already has the role, approve still succeeds (INSERT OR IGNORE).""" await _seed_user(db_path) await grant_role("user-1", "contributor", granted_by="admin-1") @@ -171,7 +161,7 @@ async def test_idempotent_user_already_has_role(self, db_path: Path): assert result is not None @pytest.mark.asyncio - async def test_returns_none_for_already_reviewed_request(self, db_path: Path): + async def test_returns_none_for_already_reviewed_request(self, db_path: str): await _seed_user(db_path) rr_id = await create_role_request("user-1", "contributor", user_note=None) await review_role_request(rr_id, "rejected", "admin-1") @@ -187,20 +177,20 @@ async def test_returns_none_for_already_reviewed_request(self, db_path: Path): class TestWithdrawRoleRequest: @pytest.mark.asyncio - async def test_withdraw_pending_request(self, db_path: Path): + async def test_withdraw_pending_request(self, db_path: str): await _seed_user(db_path) rr_id = await create_role_request("user-1", "contributor", user_note=None) assert await withdraw_role_request(rr_id, "user-1") is True @pytest.mark.asyncio - async def test_withdraw_already_approved_returns_false(self, db_path: Path): + async def test_withdraw_already_approved_returns_false(self, db_path: str): await _seed_user(db_path) rr_id = await create_role_request("user-1", "contributor", user_note=None) await review_role_request(rr_id, "approved", "admin-1") assert await withdraw_role_request(rr_id, "user-1") is False @pytest.mark.asyncio - async def test_withdraw_scoped_to_requester(self, db_path: Path): + async def test_withdraw_scoped_to_requester(self, db_path: str): await _seed_user(db_path, discord_id="user-1") rr_id = await create_role_request("user-1", "contributor", user_note=None) # Different user tries to withdraw @@ -214,36 +204,30 @@ async def test_withdraw_scoped_to_requester(self, db_path: Path): class TestCapabilityHelpers: @pytest.mark.asyncio - async def test_user_has_capability_via_granted_role(self, db_path: Path): + async def test_user_has_capability_via_granted_role(self, db_path: str): """If the user has a role that maps to a capability, returns True.""" - import aiosqlite - await _seed_user(db_path) # Seed role_permissions directly - async with aiosqlite.connect(db_path) as db: - await db.execute( - "INSERT OR IGNORE INTO role_permissions (role, capability) VALUES (?, ?)", + with pg_conn(db_path) as conn: + conn.execute( + "INSERT INTO role_permissions (role, capability) VALUES (%s, %s) ON CONFLICT DO NOTHING", ("contributor", "edit_zones"), ) - await db.commit() await grant_role("user-1", "contributor", granted_by="admin-1") assert await user_has_capability_via_db("user-1", "edit_zones") is True @pytest.mark.asyncio - async def test_user_lacks_capability_without_role(self, db_path: Path): + async def test_user_lacks_capability_without_role(self, db_path: str): await _seed_user(db_path) assert await user_has_capability_via_db("user-1", "edit_zones") is False @pytest.mark.asyncio - async def test_role_has_capability(self, db_path: Path): - import aiosqlite - - async with aiosqlite.connect(db_path) as db: - await db.execute( - "INSERT OR IGNORE INTO role_permissions (role, capability) VALUES (?, ?)", + async def test_role_has_capability(self, db_path: str): + with pg_conn(db_path) as conn: + conn.execute( + "INSERT INTO role_permissions (role, capability) VALUES (%s, %s) ON CONFLICT DO NOTHING", ("contributor", "edit_raids"), ) - await db.commit() assert await role_has_capability("contributor", "edit_raids") is True assert await role_has_capability("contributor", "nonexistent") is False @@ -255,12 +239,12 @@ async def test_role_has_capability(self, db_path: Path): class TestSetUserAccess: @pytest.mark.asyncio - async def test_returns_true_on_update(self, db_path: Path): + async def test_returns_true_on_update(self, db_path: str): await _seed_user(db_path) assert await set_user_access("user-1", "approved") is True @pytest.mark.asyncio - async def test_returns_false_for_unknown_user(self, db_path: Path): + async def test_returns_false_for_unknown_user(self, db_path: str): assert await set_user_access("ghost-user", "approved") is False @@ -271,7 +255,7 @@ async def test_returns_false_for_unknown_user(self, db_path: Path): class TestListRoleAssignments: @pytest.mark.asyncio - async def test_returns_mapping_of_user_to_roles(self, db_path: Path): + async def test_returns_mapping_of_user_to_roles(self, db_path: str): await _seed_user(db_path) await grant_role("user-1", "contributor", granted_by="admin-1") assignments = await list_role_assignments() @@ -279,7 +263,7 @@ async def test_returns_mapping_of_user_to_roles(self, db_path: Path): assert "contributor" in assignments["user-1"] @pytest.mark.asyncio - async def test_returns_empty_for_no_roles(self, db_path: Path): + async def test_returns_empty_for_no_roles(self, db_path: str): await _seed_user(db_path) assignments = await list_role_assignments() assert "user-1" not in assignments @@ -292,22 +276,22 @@ async def test_returns_empty_for_no_roles(self, db_path: Path): class TestOpenSignup: @pytest.mark.asyncio - async def test_default_new_user_is_pending(self, db_path: Path): + async def test_default_new_user_is_pending(self, db_path: str): status = await upsert_user("u-new", "New", "new", None) assert status == "pending" @pytest.mark.asyncio - async def test_open_signup_approves_new_user(self, db_path: Path): + async def test_open_signup_approves_new_user(self, db_path: str): status = await upsert_user("u-open", "Open", "open", None, open_signup=True) assert status == "approved" @pytest.mark.asyncio - async def test_admin_always_approved_even_without_open_signup(self, db_path: Path): + async def test_admin_always_approved_even_without_open_signup(self, db_path: str): status = await upsert_user("u-admin", "Admin", "admin", None, admin_ids=frozenset({"u-admin"})) assert status == "approved" @pytest.mark.asyncio - async def test_open_signup_does_not_reapprove_on_relogin(self, db_path: Path): + async def test_open_signup_does_not_reapprove_on_relogin(self, db_path: str): # First login while signup is closed → pending. await upsert_user("u-relog", "Re", "re", None, open_signup=False) # Re-login with the flag now ON must NOT auto-approve an existing user; @@ -316,7 +300,7 @@ async def test_open_signup_does_not_reapprove_on_relogin(self, db_path: Path): assert status == "pending" @pytest.mark.asyncio - async def test_approve_all_pending_clears_backlog_idempotently(self, db_path: Path): + async def test_approve_all_pending_clears_backlog_idempotently(self, db_path: str): await upsert_user("p1", "P1", "p1", None) await upsert_user("p2", "P2", "p2", None) n = await approve_all_pending() diff --git a/tests/server/test_xpac_rollover.py b/tests/server/test_xpac_rollover.py index 10f3733c..91a6c9a1 100644 --- a/tests/server/test_xpac_rollover.py +++ b/tests/server/test_xpac_rollover.py @@ -3,12 +3,10 @@ from __future__ import annotations from datetime import UTC, datetime -from pathlib import Path import pytest from backend.server.api.rankings import _apply_era_lock -from backend.server.db import init_db from backend.server.db.servers import store as servers_db from backend.server.xpac_rollover import ( XPAC_MAX_LEVEL, @@ -16,19 +14,15 @@ parse_dt, rollover_due, ) -from tests.fixtures.users_db import point_users_db_at - - -@pytest.fixture -def users_db(tmp_path) -> Path: - db = tmp_path / "users.db" - init_db(db) - return db @pytest.fixture(autouse=True) -def _stores_at_tmp(users_db: Path, monkeypatch: pytest.MonkeyPatch): - point_users_db_at(monkeypatch, users_db) +def users_db(users_schema: str) -> str: + """Isolated leased schema per test (conftest ``users_schema``), aliased + so tests can keep naming it ``users_db``. The seeds already contain the + Varsoon (default) + Wuoshi server rows at max_level 70; ``_seed`` below + UPDATEs them in place.""" + return users_schema _NOW = datetime(2026, 9, 9, 20, 0, tzinfo=UTC) From d9390aab2af3750e7bb088b8c8f77d7794b62c50 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:19:35 +0100 Subject: [PATCH 14/32] feat(simulator): Exorcise 7.1s pulse interval for both clerics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit STATIC_PULSE_INTERVALS override in spell_effects (cleric AA — applies to Templar AND Inquisitor), applied at both rotation call sites; measured live at 7.1s on both classes. (Simulator work predating the Postgres branch; its test rode along in the suite-conversion commit.) Co-Authored-By: Claude Fable 5 --- backend/eq2db/spell_effects.py | 22 ++++++++++++++++++++++ backend/server/api/character/rotation.py | 7 +++++++ 2 files changed, 29 insertions(+) diff --git a/backend/eq2db/spell_effects.py b/backend/eq2db/spell_effects.py index 0ac138dc..c94c84aa 100644 --- a/backend/eq2db/spell_effects.py +++ b/backend/eq2db/spell_effects.py @@ -607,6 +607,28 @@ class AbilityAdjustment(TypedDict): "smite wrath": 5.0, } +#: Measured pulse-cadence corrections (lowercased base name -> seconds). +#: Exorcise's maintained stream ticks every 7.1s in game where the census +#: text says 6s (Menludiir log, an ~18% slowdown no speed stat explains — +#: suspiciously close to the Perks +20% beneficial-duration stretch, which +#: would give 7.2s; a precise re-measure could reattribute this to the +#: perk). Keyed by spell name, so every class sharing the AA gets it — +#: Exorcise is in the CLERIC class tree: Templar AND Inquisitor. +STATIC_PULSE_INTERVALS: dict[str, float] = { + "exorcise": 7.1, +} + + +def apply_interval_overrides(base_name: str, components: list[DamageComponent]) -> None: + """Replace pulse-wrapper tick cadence IN PLACE with the measured value.""" + secs = STATIC_PULSE_INTERVALS.get(base_name.strip().lower()) + if secs is None: + return + for c in components: + if c.get("kind") == "dot" and c.get("from_pulse") and c.get("interval_s"): + c["interval_s"] = secs + + #: Era mod-KEY corrections (lowercased base name -> {census key: era key}): #: census text names the modern stat but the TLE effect is another one. #: Velocity reads "Increases Multi Attack" on live; on TLE it is a DPS diff --git a/backend/server/api/character/rotation.py b/backend/server/api/character/rotation.py index bae3799d..9690ed00 100644 --- a/backend/server/api/character/rotation.py +++ b/backend/server/api/character/rotation.py @@ -35,6 +35,7 @@ ParsedEffects, ProcTriggerInfo, aa_subject_of, + apply_interval_overrides, apply_mod_key_overrides, apply_text_overrides, extract_damage_pairs, @@ -376,6 +377,10 @@ def _parse_row(r_: SpellRow) -> ParsedEffects: if static is not None: parsed["components"], parsed["procs"] = static parsed["unparsed_damage"] = [] + # Measured pulse-cadence corrections (Exorcise 7.1s vs census 6s). + apply_interval_overrides( + _spells.strip_roman(row.get("name") or node.get("name") or ""), parsed["components"] + ) level = row.get("level") or 0 lasts = parsed["lasts_for_s"] components, procs = _build_component_models(parsed, lasts, level) @@ -681,6 +686,8 @@ def _build_abilities_sync( if class_static is not None: parsed["components"] = class_static parsed["unparsed_damage"] = [] + # Measured pulse-cadence corrections (name-keyed; Exorcise 7.1s). + apply_interval_overrides(_spells.strip_roman(r.get("name") or ""), parsed["components"]) raw_duration = m.get("spell_duration") duration_s = (raw_duration / SPELL_DURATION_DIVISOR) if raw_duration else None From 352a55891cff231cec15509bb60b74e0b42fae39 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:22:37 +0100 Subject: [PATCH 15/32] feat(pg): reviewed parses-family schema (0002) + generic fixture re-provisioning 0002_parses.sql: encounters/combatants/damage_types/attack_types/ ingest_log/tamper_reports translated per the type policy (identity bigints, epoch bigints, double precision, 0/1 ints, payload_json stays byte-exact text). Review items: detail_pruned_at on encounters + partial sweep index (tiered retention: curated raid 30d / group-instance 14d / other 7d); rankings cover becomes key(encounter_id, damage DESC) + INCLUDE payload with aggressive combatants autovacuum (re-benchmark at rehearsal); new (world, lower(guild_name), started_at DESC) index for the NOCASE rewrites. Test provisioning now derives family schemas from the migration headers, so new family files re-provision cleanly. Co-Authored-By: Claude Fable 5 --- db/migrations/0002_parses.sql | 214 ++++++++++++++++++++++++++++++++++ tests/fixtures/pg.py | 16 ++- 2 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 db/migrations/0002_parses.sql diff --git a/db/migrations/0002_parses.sql b/db/migrations/0002_parses.sql new file mode 100644 index 00000000..b3438a30 --- /dev/null +++ b/db/migrations/0002_parses.sql @@ -0,0 +1,214 @@ +create schema if not exists parses; +set search_path to parses, public; + +-- Reviewed translation of backend/server/parses/db.sql (SQLite) with the +-- post-ALTER columns folded in. Policy (plan unified-swimming-kazoo): +-- identity bigints, blanket bigint for epochs/counters (damage totals +-- exceed int4), double precision for REAL, 0/1 flags stay integers, +-- JSON-ish payloads stay text byte-exact. FKs are declared and enforced +-- (SQLite never enforced them here either, but the copy script audits +-- orphans before load). + +CREATE TABLE encounters ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + world text NOT NULL DEFAULT 'Varsoon', + act_encid text NOT NULL, + title text NOT NULL, + zone text, + started_at bigint NOT NULL, -- unix seconds, UTC + ended_at bigint NOT NULL, + duration_s bigint NOT NULL, + total_damage bigint NOT NULL DEFAULT 0, + encdps double precision NOT NULL DEFAULT 0, + kills bigint NOT NULL DEFAULT 0, + deaths bigint NOT NULL DEFAULT 0, + -- ACT's GetEncounterSuccessLevel(): 0=unknown, 1=win, 2=loss, 3=mixed. + success_level integer NOT NULL DEFAULT 0, + source_dsn text NOT NULL, + uploaded_by text NOT NULL DEFAULT 'local', + guild_name text, + ingested_at bigint NOT NULL, + -- Soft-delete marker (unix seconds). NULL = visible. + hidden_at bigint, + hidden_by text, + -- EQ2Parser client_warnings provenance stamps (JSON text, byte-exact). + client_warnings text, + -- REVIEW (tiered detail retention, user decision 2026-10-04): set when + -- the cleanup sweep drops this encounter's attack_types/damage_types + -- rows (curated raid zones 30d, curated group-instance 14d, any other + -- kept fight 7d). encounters+combatants live forever; the parse detail + -- page shows a "breakdown pruned" notice instead of empty tables. + detail_pruned_at bigint, + UNIQUE (world, act_encid) +); + +CREATE TABLE combatants ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + encounter_id bigint NOT NULL REFERENCES encounters(id) ON DELETE CASCADE, + name text NOT NULL, + ally integer NOT NULL DEFAULT 0, -- 0/1 (ACT's 'T'/'F') + started_at bigint NOT NULL DEFAULT 0, + ended_at bigint NOT NULL DEFAULT 0, + duration_s bigint NOT NULL DEFAULT 0, + damage bigint NOT NULL DEFAULT 0, + damage_perc double precision NOT NULL DEFAULT 0, + kills bigint NOT NULL DEFAULT 0, + healed bigint NOT NULL DEFAULT 0, + healed_perc double precision NOT NULL DEFAULT 0, + crit_heals bigint NOT NULL DEFAULT 0, + heals bigint NOT NULL DEFAULT 0, + cure_dispels bigint NOT NULL DEFAULT 0, + power_drain bigint NOT NULL DEFAULT 0, + power_replenish bigint NOT NULL DEFAULT 0, + dps double precision NOT NULL DEFAULT 0, + encdps double precision NOT NULL DEFAULT 0, + enchps double precision NOT NULL DEFAULT 0, + hits bigint NOT NULL DEFAULT 0, + crit_hits bigint NOT NULL DEFAULT 0, + blocked bigint NOT NULL DEFAULT 0, + misses bigint NOT NULL DEFAULT 0, + swings bigint NOT NULL DEFAULT 0, + heals_taken bigint NOT NULL DEFAULT 0, + damage_taken bigint NOT NULL DEFAULT 0, + deaths bigint NOT NULL DEFAULT 0, + to_hit double precision NOT NULL DEFAULT 0, + crit_dam_perc double precision NOT NULL DEFAULT 0, + crit_heal_perc double precision NOT NULL DEFAULT 0, + crit_types text, + threat_str text, + threat_delta bigint NOT NULL DEFAULT 0, + -- Identity snapshot frozen at ingest. NULL for pets/NPCs and players + -- we couldn't resolve at upload time. + level integer, + guild_name text, + cls text, + ilvl double precision, + -- Tri-state classification: NULL = not yet classified, 0/1 = decided. + is_player integer, + UNIQUE (encounter_id, name) +); +-- Churny table + the covering index below needs a clean visibility map for +-- index-only scans — vacuum far more eagerly than the default 20%. +ALTER TABLE combatants SET (autovacuum_vacuum_scale_factor = 0.02, autovacuum_analyze_scale_factor = 0.02); + +CREATE TABLE damage_types ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + combatant_id bigint NOT NULL REFERENCES combatants(id) ON DELETE CASCADE, + grouping_label text, + damage_type text NOT NULL, + started_at bigint NOT NULL DEFAULT 0, + ended_at bigint NOT NULL DEFAULT 0, + duration_s bigint NOT NULL DEFAULT 0, + damage bigint NOT NULL DEFAULT 0, + encdps double precision NOT NULL DEFAULT 0, + char_dps double precision NOT NULL DEFAULT 0, + dps double precision NOT NULL DEFAULT 0, + average double precision NOT NULL DEFAULT 0, + median bigint NOT NULL DEFAULT 0, + min_hit bigint NOT NULL DEFAULT 0, + max_hit bigint NOT NULL DEFAULT 0, + hits bigint NOT NULL DEFAULT 0, + crit_hits bigint NOT NULL DEFAULT 0, + blocked bigint NOT NULL DEFAULT 0, + misses bigint NOT NULL DEFAULT 0, + swings bigint NOT NULL DEFAULT 0, + to_hit double precision NOT NULL DEFAULT 0, + average_delay double precision NOT NULL DEFAULT 0, + crit_perc double precision NOT NULL DEFAULT 0, + crit_types text, + UNIQUE (combatant_id, damage_type) +); + +CREATE TABLE attack_types ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + combatant_id bigint NOT NULL REFERENCES combatants(id) ON DELETE CASCADE, + victim text, + swing_type integer NOT NULL DEFAULT 0, + attack_name text NOT NULL, + started_at bigint NOT NULL DEFAULT 0, + ended_at bigint NOT NULL DEFAULT 0, + duration_s bigint NOT NULL DEFAULT 0, + damage bigint NOT NULL DEFAULT 0, + encdps double precision NOT NULL DEFAULT 0, + char_dps double precision NOT NULL DEFAULT 0, + dps double precision NOT NULL DEFAULT 0, + average double precision NOT NULL DEFAULT 0, + median bigint NOT NULL DEFAULT 0, + min_hit bigint NOT NULL DEFAULT 0, + max_hit bigint NOT NULL DEFAULT 0, + resist text, + hits bigint NOT NULL DEFAULT 0, + crit_hits bigint NOT NULL DEFAULT 0, + blocked bigint NOT NULL DEFAULT 0, + misses bigint NOT NULL DEFAULT 0, + swings bigint NOT NULL DEFAULT 0, + to_hit double precision NOT NULL DEFAULT 0, + average_delay double precision NOT NULL DEFAULT 0, + crit_perc double precision NOT NULL DEFAULT 0, + crit_types text, + UNIQUE (combatant_id, swing_type, attack_name) +); + +CREATE TABLE ingest_log ( + world text NOT NULL DEFAULT 'Varsoon', + act_encid text NOT NULL, + encounter_id bigint NOT NULL REFERENCES encounters(id) ON DELETE CASCADE, + ingested_at bigint NOT NULL, + source_dsn text NOT NULL, + PRIMARY KEY (world, act_encid) +); + +CREATE TABLE tamper_reports ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + world text NOT NULL DEFAULT 'Varsoon', + act_encid text NOT NULL, + title text NOT NULL, + zone text, + started_at bigint NOT NULL, -- unix seconds, UTC + ended_at bigint NOT NULL, + duration_s bigint NOT NULL, + total_damage bigint NOT NULL DEFAULT 0, + encdps double precision NOT NULL DEFAULT 0, + reason text NOT NULL, + reported_at bigint NOT NULL, + uploader_logger_name text NOT NULL DEFAULT '', + uploader_discord_id text NOT NULL DEFAULT '', + uploader_discord_name text NOT NULL DEFAULT '', + guild_name text, + -- Byte-exact copy of the rejected payload (forensics) — stays text, + -- never jsonb (jsonb would normalise key order/whitespace). + payload_json text NOT NULL, + acknowledged_at bigint, + acknowledged_by text +); + +-- Indexes (ported verbatim unless annotated) ------------------------------- +CREATE INDEX idx_encounters_started_desc ON encounters (started_at DESC); +CREATE INDEX idx_encounters_zone ON encounters (zone); +CREATE INDEX idx_encounters_world ON encounters (world, started_at DESC); +CREATE INDEX idx_encounters_uploaded_by ON encounters (uploaded_by, started_at DESC); +-- REVIEW: replaces the SQLite "COLLATE NOCASE on guild_name" scans — the +-- attendance/list reads rewrite to lower(guild_name) = lower(%s). +CREATE INDEX idx_encounters_world_guild_lower ON encounters (world, lower(guild_name), started_at DESC); +-- REVIEW: the tiered-retention sweep's candidate scan (old encounters whose +-- breakdown rows still exist). Partial: rows already pruned drop out. +CREATE INDEX idx_encounters_detail_prune ON encounters (started_at) WHERE detail_pruned_at IS NULL; +CREATE INDEX idx_combatants_encounter ON combatants (encounter_id); +CREATE INDEX idx_combatants_name ON combatants (name); +CREATE INDEX idx_combatants_ally ON combatants (encounter_id, ally); +CREATE INDEX idx_combatants_encounter_is_player ON combatants (encounter_id, is_player); +-- REVIEW: the rankings cover moves to key(encounter_id, damage DESC) + +-- INCLUDE payload — same index-only scan, smaller btree keys. Re-benchmark +-- the rankings rebuild on real data at rehearsal (837s-incident guard). +CREATE INDEX idx_combatants_rankings_cover ON combatants (encounter_id, damage DESC) + INCLUDE (name, ally, is_player, cls, level, ilvl, guild_name, encdps, enchps, healed, deaths); +CREATE INDEX idx_damage_types_combatant ON damage_types (combatant_id); +CREATE INDEX idx_attack_types_combatant ON attack_types (combatant_id); +CREATE INDEX idx_attack_types_damage_desc ON attack_types (combatant_id, damage DESC); +CREATE INDEX idx_tamper_reports_unack ON tamper_reports (reported_at DESC) WHERE acknowledged_at IS NULL; +CREATE INDEX idx_tamper_reports_reporter ON tamper_reports (uploader_discord_id, reported_at DESC); +CREATE INDEX idx_tamper_reports_world_reported ON tamper_reports (world, reported_at DESC); + +-- seeds +-- (none — parses is pure runtime data; the marker keeps the scratch-schema +-- leaser's reset contract uniform across family files) diff --git a/tests/fixtures/pg.py b/tests/fixtures/pg.py index 775e8e96..1a047c81 100644 --- a/tests/fixtures/pg.py +++ b/tests/fixtures/pg.py @@ -102,14 +102,22 @@ def provision_for_session() -> None: print("[pg-fixtures] another pytest session holds the test database — waiting for it to finish...") _session_lock_conn.execute("SELECT pg_advisory_lock(%s)", (_SESSION_LOCK_ID,)) + # Family schemas are derived from the migration files' two-line headers, + # so adding 0003_census.sql etc. needs no fixture change. + families: list[str] = [] + for path in pg_migrate.migration_files(): + first = path.read_text(encoding="utf-8").splitlines()[0].strip().lower() + if first.startswith("create schema if not exists "): + families.append(first.removeprefix("create schema if not exists ").rstrip(";").strip()) with pg.connection() as conn: # Drop the ledger so pg_migrate re-applies everything, then the # session schemas and any scratch schemas a crashed run left over. conn.execute("DROP TABLE IF EXISTS public.schema_migrations") - conn.execute("DROP SCHEMA IF EXISTS users CASCADE") - rows = conn.execute("SELECT nspname FROM pg_namespace WHERE nspname LIKE 'users\\_s%'").fetchall() - for r in rows: - conn.execute(f'DROP SCHEMA IF EXISTS "{r["nspname"]}" CASCADE') + for fam in families: + conn.execute(pgsql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(pgsql.Identifier(fam))) + rows = conn.execute("SELECT nspname FROM pg_namespace WHERE nspname LIKE %s", (fam + r"\_s%",)).fetchall() + for r in rows: + conn.execute(pgsql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(pgsql.Identifier(r["nspname"]))) conn.commit() pg_migrate.run() From d5da39b9c91f929bee5780a3068d14a43ffd3e44 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:24:05 +0100 Subject: [PATCH 16/32] feat(pg): PgCatalogue + pooled caller-owned connection seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pg.getconn()/putconn() give the catalogue families a caller-owned sync checkout (pool-if-open-else-direct). PgCatalogue.init_db() keeps its name but creates nothing — it hands back a PgConnProxy scoped to the family schema: sqlite3.Connection surface (execute/commit/close, txn- scope `with`), close() returns to the pool, dict rows, fetchval() for the old positional fetchone()[0] sites. Preserves the ~60 caller-owns-connection sites' shape across parses/census/zones/raids. Co-Authored-By: Claude Fable 5 --- backend/db_catalogue.py | 76 +++++++++++++++++++++++++++++++++++++++++ backend/pg.py | 24 +++++++++++++ 2 files changed, 100 insertions(+) diff --git a/backend/db_catalogue.py b/backend/db_catalogue.py index f50b93d4..af6cdd84 100644 --- a/backend/db_catalogue.py +++ b/backend/db_catalogue.py @@ -362,3 +362,79 @@ async def _db(self, *, row_factory: bool = True) -> AsyncIterator[Any]: async with pg.aconnection() as conn: await conn.execute(pg.search_path_sql(self.schema)) yield conn + + +class PgConnProxy: + """A caller-owned Postgres connection scoped to one family schema. + + Mimics the sqlite3.Connection surface the catalogue callers actually + use — ``execute``/``executemany``/``commit``/``rollback``/``close``, + plus the sqlite transaction-scope ``with`` shape (commit on clean + exit, rollback on exception, THEN return to the pool — sqlite's + ``with`` kept the handle open, but no converted caller reuses it). + ``close()`` returns the connection to the pool rather than killing a + pooled connection. Rows are dicts (dict_row).""" + + def __init__(self, schema: str) -> None: + from backend import pg + + self._pg = pg + self._conn: Any = pg.getconn() + self._conn.execute(pg.search_path_sql(schema)) + + def execute(self, sql: str, params: Any = None) -> Any: + return self._conn.execute(sql, params) + + def executemany(self, sql: str, params_seq: Any) -> Any: + cur = self._conn.cursor() + cur.executemany(sql, params_seq) + return cur + + def cursor(self) -> Any: + return self._conn.cursor() + + def commit(self) -> None: + self._conn.commit() + + def rollback(self) -> None: + self._conn.rollback() + + def close(self) -> None: + conn, self._conn = self._conn, None + if conn is not None: + self._pg.putconn(conn) + + def __enter__(self) -> PgConnProxy: + return self + + def __exit__(self, exc_type: object, exc: object, tb: object) -> None: + try: + if self._conn is not None: + if exc_type is None: + self._conn.commit() + else: + self._conn.rollback() + finally: + self.close() + + +class PgCatalogue(SchemaBound): + """Sync Postgres analogue of :class:`BaseCatalogue` for the + caller-owns-connection families (parses, census, zones, raids). + + ``init_db()`` KEEPS ITS NAME but creates nothing — schema DDL is owned + by db/migrations/. It returns an open :class:`PgConnProxy` scoped to + ``self.schema``, preserving the ``conn = store.init_db(); …; + conn.close()`` caller shape across ~60 sites.""" + + def init_db(self) -> PgConnProxy: + return PgConnProxy(self.schema) + + @staticmethod + def fetchval(cur: Any) -> Any: + """First column of the first row (or None) — the dict-row + replacement for sqlite's positional ``fetchone()[0]``.""" + row = cur.fetchone() + if row is None: + return None + return next(iter(row.values())) diff --git a/backend/pg.py b/backend/pg.py index 97715e39..7e6d052e 100644 --- a/backend/pg.py +++ b/backend/pg.py @@ -105,6 +105,30 @@ def connection() -> Iterator[Any]: yield conn +def getconn() -> Any: + """Caller-owned sync checkout — ALWAYS pair with :func:`putconn`. + Pooled when the lifespan opened pools, else a direct connection. + For scoped work prefer the :func:`connection` contextmanager; this + exists for the catalogue families' caller-owns-connection pattern + (PgCatalogue.init_db in backend/db_catalogue.py).""" + if _sync_pool is not None: + return _sync_pool.getconn() + return psycopg.connect(dsn(), row_factory=dict_row) # type: ignore[arg-type] + + +def putconn(conn: Any) -> None: + """Return a :func:`getconn` connection. Pool-origin connections go back + to the pool (psycopg_pool resets them); direct ones just close. The + pool-closed-since race degrades to close().""" + if _sync_pool is not None: + try: + _sync_pool.putconn(conn) + except ValueError: + conn.close() + return + conn.close() + + def search_path_sql(schema: str) -> pgsql.Composed: """The one statement stores run at checkout — composed so a schema name can never inject (test scratch schemas are generated names).""" From ab1a3549aa67a6ffd436ff5eaa544eb806173866 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:28:17 +0100 Subject: [PATCH 17/32] =?UTF-8?q?feat(pg):=20ParsesStore=20on=20psycopg=20?= =?UTF-8?q?=E2=80=94=20PgCatalogue,=20ANY()=20batching,=20retention=20help?= =?UTF-8?q?ers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit db.sql is DML-only now (DDL owns 0002_parses.sql; the SQLite schema/ ALTER/rebuild machinery is deleted): named inserts use %(param)s against the models' as_db_params dicts, IN-list {placeholders} composition is replaced by = ANY(%s) (the 500-id chunking in get_combatants_for_ encounters is gone), instr->strpos, IFNULL->COALESCE, COUNT aliases. db.py: PgCatalogue base (init_db returns a pooled schema-scoped proxy), dict rows, explicit commits replace sqlite `with conn:` scopes, and the new tiered-retention helpers (select_detail_prune_candidates / prune_encounter_detail stamping detail_pruned_at). Co-Authored-By: Claude Fable 5 --- backend/server/parses/db.py | 461 +++++++++++-------------------- backend/server/parses/db.sql | 520 ++++++++--------------------------- 2 files changed, 271 insertions(+), 710 deletions(-) diff --git a/backend/server/parses/db.py b/backend/server/parses/db.py index 004d9388..ce8a1bc5 100644 --- a/backend/server/parses/db.py +++ b/backend/server/parses/db.py @@ -1,14 +1,12 @@ """ -Normalized SQLite store for ingested ACT parses. +Normalized Postgres store (``parses`` schema) for ingested ACT parses. All behaviour lives on :class:`ParsesStore` (the catalogue convention — -see backend/db_catalogue.py): ``store.init_db()`` returns a connection -with WAL/foreign-keys enabled (schema + idempotent ``_MIGRATIONS`` applied -via the base template method); the conn-taking insert/lookup helpers are -staticmethods — callers batch several operations per connection. - -Lives at `data/parses/parses.db` by default. Override with the -`DB_PARSES_PATH` env var. +see backend/db_catalogue.py): ``store.init_db()`` returns a pooled +connection scoped to the family schema (PgConnProxy — ``close()`` returns +it to the pool); the conn-taking insert/lookup helpers are staticmethods — +callers batch several operations per connection. Schema DDL is owned by +db/migrations/0002_parses.sql. Schema reflects the real columns ACT exports at AttackType depth — the plugin's PayloadBuilder (in the EQ2LexiconACTPlugin repo) is the upstream @@ -17,12 +15,10 @@ from __future__ import annotations -import sqlite3 from enum import IntEnum -from pathlib import Path +from typing import Any -from backend.db_catalogue import BaseCatalogue -from backend.db_helpers import resolve_db_path +from backend.db_catalogue import PgCatalogue from backend.server.parses.models import AttackType, Combatant, CombatantSnapshot, DamageType, Encounter from backend.sql_loader import load_sql @@ -31,44 +27,8 @@ # Reused for combatants with no resolved identity snapshot — stores NULLs. _EMPTY_SNAPSHOT = CombatantSnapshot() -# --------------------------------------------------------------------------- -# Config -# --------------------------------------------------------------------------- - - -DB_PATH: Path = resolve_db_path("DB_PARSES_PATH", "parses", "parses.db") - - -# Schema (CREATE TABLE / INDEX) lives in db.sql; init_db runs each block. -# Back-compat aliases — keep tests + any external callers that imported -# _CREATE_* directly off this module working without an import-shape change. -_CREATE_ENCOUNTERS = _SQL["schema_encounters"] -_CREATE_COMBATANTS = _SQL["schema_combatants"] -_CREATE_DAMAGE_TYPES = _SQL["schema_damage_types"] -_CREATE_ATTACK_TYPES = _SQL["schema_attack_types"] -_CREATE_INGEST_LOG = _SQL["schema_ingest_log"] -_CREATE_TAMPER_REPORTS = _SQL["schema_tamper_reports"] -# `indexes_all` is one multi-statement block; split on semicolons to keep the -# legacy list shape that test fixtures iterate. -_CREATE_INDEXES = [s.strip() + ";" for s in _SQL["indexes_all"].split(";") if s.strip()] - -# Idempotent ALTER migrations — each statement loaded from db.sql. init_db -# loops the list, swallowing OperationalError so re-runs on an up-to-date DB -# are no-ops. Order is significant: column-dependent migrations (e.g. an -# index on a new column) MUST come after the ADD COLUMN they depend on. -_MIGRATIONS: list[str] = [ - _SQL["alter_encounters_add_uploaded_by"], - _SQL["alter_encounters_add_guild_name"], - _SQL["alter_encounters_add_success_level"], - _SQL["alter_combatants_add_level"], - _SQL["alter_combatants_add_guild_name"], - _SQL["alter_combatants_add_cls"], - _SQL["alter_combatants_add_ilvl"], - _SQL["alter_encounters_add_hidden_at"], - _SQL["alter_combatants_add_is_player"], - _SQL["alter_encounters_add_client_warnings"], - _SQL["alter_encounters_add_hidden_by"], -] +#: Postgres schema the parses family lives in (db/migrations/0002_parses.sql). +SCHEMA = "parses" # ACT swing_type semantics confirmed against real EQ2 data: @@ -96,135 +56,24 @@ class SwingType(IntEnum): PROC = 100 -_DAMAGE_SWING_TYPES = (SwingType.MELEE, SwingType.NONMELEE) -_HEAL_SWING_TYPES = (SwingType.HEAL,) -_CURE_SWING_TYPES = (SwingType.CURE,) -_THREAT_SWING_TYPES = (SwingType.PROC,) # callers should additionally filter type != 'All' +_DAMAGE_SWING_TYPES = [int(SwingType.MELEE), int(SwingType.NONMELEE)] +_HEAL_SWING_TYPES = [int(SwingType.HEAL)] +_CURE_SWING_TYPES = [int(SwingType.CURE)] +_THREAT_SWING_TYPES = [int(SwingType.PROC)] # callers should additionally filter type != 'All' -class ParsesStore(BaseCatalogue): - """Read/write access to one parses.db file (uploaded ACT encounters). +class ParsesStore(PgCatalogue): + """Read/write access to the parses schema (uploaded ACT encounters). The catalogue convention (see backend/db_catalogue.py): the shared module-level ``store`` instance is the runtime entry point (consumers alias it ``parses_db``); the conn-taking helpers are staticmethods — callers batch several operations per connection/transaction. Tests - construct ``ParsesStore(tmp_db)``. + construct ``ParsesStore(scratch_schema)`` or re-point ``store.schema``. """ - FOREIGN_KEYS = True - - # parses.db predates the shared _meta table; rows carry their own - # timestamps and litestream owns backup provenance. - CREATE_META = False - - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) - - # --------------------------------------------------------------------------- - # DB management - # --------------------------------------------------------------------------- - - def _create_schema(self, conn: sqlite3.Connection) -> None: - conn.execute(_SQL["schema_encounters"]) - conn.execute(_SQL["schema_combatants"]) - conn.execute(_SQL["schema_damage_types"]) - conn.execute(_SQL["schema_attack_types"]) - conn.execute(_SQL["schema_ingest_log"]) - conn.execute(_SQL["schema_tamper_reports"]) - self._apply_migrations(conn, _MIGRATIONS) - self._migrate_attack_types_unique(conn) - self._migrate_encounters_add_world(conn) - self._migrate_ingest_log_add_world(conn) - conn.executescript(_SQL["indexes_all"]) - - @staticmethod - def _migrate_attack_types_unique(conn: sqlite3.Connection) -> None: - """Recreate attack_types if the legacy UNIQUE(combatant_id, attack_name) - constraint is still in place. The natural key needs swing_type too — - spells like Cleanse legitimately appear under both damage and heal.""" - rows = conn.execute(_SQL["migrate_check_attack_types_indexes"]).fetchall() - target = ["combatant_id", "swing_type", "attack_name"] - for (idx_name,) in rows: - cols = [r[2] for r in conn.execute(_SQL["pragma_index_info"].format(idx_name=idx_name)).fetchall()] - if cols == target: - return # already migrated - # Commit any pending implicit transaction so `with conn:` can scope a - # fresh atomic one around the table swap. - conn.commit() - with conn: - conn.execute( - _SQL["schema_attack_types"].replace( - "CREATE TABLE IF NOT EXISTS attack_types", - "CREATE TABLE attack_types_new", - ) - ) - conn.execute(_SQL["migrate_attack_types_insert_into_new"]) - conn.execute(_SQL["migrate_attack_types_drop_old"]) - conn.execute(_SQL["migrate_attack_types_rename"]) - - @staticmethod - def _migrate_encounters_add_world(conn: sqlite3.Connection) -> None: - """Add the `world` column to encounters and change the uniqueness key from - the single-column UNIQUE on act_encid to UNIQUE(world, act_encid). - - SQLite can't ALTER a uniqueness constraint in place, so we use the standard - table-rebuild pattern: rename old → create new → INSERT … SELECT → drop old. - - Guard: skip if the `world` column already exists (idempotent). - FK safety: combatants.encounter_id references encounters(id). The rebuild - preserves all `id` values via an explicit column list (including the - original INTEGER PRIMARY KEY AUTOINCREMENT sequence), so child rows remain - valid after the swap. PRAGMA foreign_keys is turned OFF for the duration of - the rebuild so SQLite does not object while encounters_old is the target; - it is re-enabled immediately after.""" - cols = [r[1] for r in conn.execute(_SQL["pragma_table_info_encounters"]).fetchall()] - if "world" in cols: - return # already migrated - - conn.commit() - conn.execute(_SQL["pragma_foreign_keys_off"]) - conn.execute(_SQL["pragma_legacy_alter_table_on"]) - try: - with conn: - conn.execute(_SQL["migrate_encounters_rename_old"]) - conn.execute( - _SQL["schema_encounters"].replace( - "CREATE TABLE IF NOT EXISTS encounters", "CREATE TABLE encounters" - ) - ) - # Copy all existing rows, backfilling world = 'Varsoon'. - conn.execute(_SQL["migrate_encounters_copy_from_old"]) - conn.execute(_SQL["migrate_encounters_drop_old"]) - finally: - conn.execute(_SQL["pragma_legacy_alter_table_off"]) - conn.execute(_SQL["pragma_foreign_keys_on"]) - - @staticmethod - def _migrate_ingest_log_add_world(conn: sqlite3.Connection) -> None: - """Add `world` to ingest_log and change PK from act_encid alone to - (world, act_encid). Same rebuild pattern as encounters; guard on 'world' - column presence.""" - cols = [r[1] for r in conn.execute(_SQL["pragma_table_info_ingest_log"]).fetchall()] - if "world" in cols: - return # already migrated - - conn.commit() - conn.execute(_SQL["pragma_foreign_keys_off"]) - conn.execute(_SQL["pragma_legacy_alter_table_on"]) - try: - with conn: - conn.execute(_SQL["migrate_ingest_log_rename_old"]) - conn.execute( - _SQL["schema_ingest_log"].replace( - "CREATE TABLE IF NOT EXISTS ingest_log", "CREATE TABLE ingest_log" - ) - ) - conn.execute(_SQL["migrate_ingest_log_copy_from_old"]) - conn.execute(_SQL["migrate_ingest_log_drop_old"]) - finally: - conn.execute(_SQL["pragma_legacy_alter_table_off"]) - conn.execute(_SQL["pragma_foreign_keys_on"]) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) # --------------------------------------------------------------------------- # Insert helpers @@ -232,7 +81,7 @@ def _migrate_ingest_log_add_world(conn: sqlite3.Connection) -> None: @staticmethod def insert_encounter( - conn: sqlite3.Connection, + conn: Any, enc: Encounter, *, source_dsn: str, @@ -256,11 +105,11 @@ def insert_encounter( ), ) row = cur.fetchone() - return int(row[0]) if row else 0 + return int(row["id"]) if row else 0 @staticmethod def insert_combatants_bulk( - conn: sqlite3.Connection, + conn: Any, encounter_id: int, combatants: list[Combatant], snapshots: dict[str, CombatantSnapshot] | None = None, @@ -276,12 +125,12 @@ def insert_combatants_bulk( c.as_db_params(encounter_id=encounter_id, snapshot=snap), ) row = cur.fetchone() - name_to_id[c.name] = int(row[0]) if row else 0 + name_to_id[c.name] = int(row["id"]) if row else 0 return name_to_id @staticmethod def update_combatant_snapshots( - conn: sqlite3.Connection, + conn: Any, encounter_id: int, snapshots: dict[str, CombatantSnapshot], ) -> int: @@ -291,17 +140,17 @@ def update_combatant_snapshots( if not snapshots: return 0 n = 0 - with conn: - for name, snap in snapshots.items(): - cur = conn.execute( - _SQL["update_combatant_snapshot"], - (snap.level, snap.guild_name, snap.cls, snap.ilvl, encounter_id, name), - ) - n += cur.rowcount + for name, snap in snapshots.items(): + cur = conn.execute( + _SQL["update_combatant_snapshot"], + (snap.level, snap.guild_name, snap.cls, snap.ilvl, encounter_id, name), + ) + n += cur.rowcount + conn.commit() return n @staticmethod - def update_combatant_is_player(conn: sqlite3.Connection, classification: dict[int, bool]) -> None: + def update_combatant_is_player(conn: Any, classification: dict[int, bool]) -> None: """Bulk UPDATE the per-combatant is_player flag. Called from: @@ -313,37 +162,37 @@ def update_combatant_is_player(conn: sqlite3.Connection, classification: dict[in and transaction scope.""" if not classification: return - conn.executemany( + cur = conn.cursor() + cur.executemany( _SQL["update_combatant_is_player"], [(1 if v else 0, k) for k, v in classification.items()], ) @staticmethod - def invalidate_is_player_cache_with_conn(conn: sqlite3.Connection) -> None: + def invalidate_is_player_cache_with_conn(conn: Any) -> None: """Mark every combatant row for lazy re-classification on next read. Variant that accepts an existing connection (used by tests + by the - rankings cache-invalidation hook to share the parses.db connection).""" + rankings cache-invalidation hook to share the parses connection).""" conn.execute(_SQL["invalidate_is_player_cache"]) def invalidate_is_player_cache(self) -> None: """Mark every combatant row for lazy re-classification on next read. - Production caller (opens its own connection). + Production caller (checks out its own connection). Called by web/routes/rankings.py:invalidate_zones_cache so that a curator zone-edit propagates to the existing parses without a separate backfill — the next read of each encounter re-classifies - against the updated zone trees. - - Brute-force table-wide invalidation is fine at current data size - (test parses only as of 2026-05-30). If the parses corpus grows past - ~10k encounters and this becomes painful, swap for a per-zone-targeted - invalidation using an is_player_computed_at timestamp.""" - with sqlite3.connect(self.path) as conn: + against the updated zone trees.""" + conn = self.init_db() + try: self.invalidate_is_player_cache_with_conn(conn) + conn.commit() + finally: + conn.close() @staticmethod def insert_damage_types_bulk( - conn: sqlite3.Connection, + conn: Any, combatant_name_to_id: dict[str, int], damage_types: list[DamageType], ) -> int: @@ -355,12 +204,13 @@ def insert_damage_types_bulk( for dt in damage_types if dt.combatant_name in combatant_name_to_id ] - conn.executemany(_SQL["insert_damage_type"], rows) + cur = conn.cursor() + cur.executemany(_SQL["insert_damage_type"], rows) return len(rows) @staticmethod def insert_attack_types_bulk( - conn: sqlite3.Connection, + conn: Any, combatant_name_to_id: dict[str, int], attack_types: list[AttackType], ) -> int: @@ -372,12 +222,13 @@ def insert_attack_types_bulk( for at in attack_types if at.combatant_name in combatant_name_to_id ] - conn.executemany(_SQL["insert_attack_type"], rows) + cur = conn.cursor() + cur.executemany(_SQL["insert_attack_type"], rows) return len(rows) @staticmethod def mark_ingested( - conn: sqlite3.Connection, + conn: Any, act_encid: str, encounter_id: int, *, @@ -395,7 +246,7 @@ def mark_ingested( # --------------------------------------------------------------------------- @staticmethod - def is_ingested(conn: sqlite3.Connection, act_encid: str, world: str = "Varsoon") -> bool: + def is_ingested(conn: Any, act_encid: str, world: str = "Varsoon") -> bool: row = conn.execute( _SQL["check_is_ingested"], (world, act_encid), @@ -403,8 +254,7 @@ def is_ingested(conn: sqlite3.Connection, act_encid: str, world: str = "Varsoon" return row is not None @staticmethod - def find_encounter_by_act_encid(conn: sqlite3.Connection, act_encid: str, world: str = "Varsoon") -> dict | None: - conn.row_factory = sqlite3.Row + def find_encounter_by_act_encid(conn: Any, act_encid: str, world: str = "Varsoon") -> dict | None: row = conn.execute( _SQL["find_encounter_by_act_encid"], (world, act_encid), @@ -413,12 +263,11 @@ def find_encounter_by_act_encid(conn: sqlite3.Connection, act_encid: str, world: @staticmethod def recent_encounters( - conn: sqlite3.Connection, + conn: Any, limit: int = 20, zone: str | None = None, world: str = "Varsoon", ) -> list[dict]: - conn.row_factory = sqlite3.Row if zone: rows = conn.execute( _SQL["recent_encounters_by_zone"], @@ -433,7 +282,7 @@ def recent_encounters( @staticmethod def list_encounters_for_admin( - conn: sqlite3.Connection, + conn: Any, *, search: str | None = None, limit: int = 200, @@ -453,16 +302,15 @@ def list_encounters_for_admin( unix timestamp (pass the previous page's last started_at). ``hidden_only`` narrows to soft-deleted rows — the restore workflow after a mistaken guild-wide delete.""" - conn.row_factory = sqlite3.Row clauses: list[str] = [] params: list = [] if world is not None: - clauses.append("e.world = ?") + clauses.append("e.world = %s") params.append(world) if hidden_only: clauses.append("e.hidden_at IS NOT NULL") if before is not None: - clauses.append("e.started_at < ?") + clauses.append("e.started_at < %s") params.append(before) if search: like = f"%{search.lower()}%" @@ -472,13 +320,14 @@ def list_encounters_for_admin( id_term = search.strip().lstrip("#") if id_term.isdigit(): clauses.append( - "(e.id = ? OR LOWER(title) LIKE ? OR LOWER(IFNULL(uploaded_by, '')) LIKE ?" - " OR LOWER(IFNULL(guild_name, '')) LIKE ?)" + "(e.id = %s OR LOWER(title) LIKE %s OR LOWER(COALESCE(uploaded_by, '')) LIKE %s" + " OR LOWER(COALESCE(guild_name, '')) LIKE %s)" ) params += [int(id_term), like, like, like] else: clauses.append( - "(LOWER(title) LIKE ? OR LOWER(IFNULL(uploaded_by, '')) LIKE ? OR LOWER(IFNULL(guild_name, '')) LIKE ?)" + "(LOWER(title) LIKE %s OR LOWER(COALESCE(uploaded_by, '')) LIKE %s" + " OR LOWER(COALESCE(guild_name, '')) LIKE %s)" ) params += [like, like, like] where = ("WHERE " + " AND ".join(clauses)) if clauses else "" @@ -486,54 +335,51 @@ def list_encounters_for_admin( return [dict(r) for r in conn.execute(sql, [*params, limit]).fetchall()] @staticmethod - def delete_encounter(conn: sqlite3.Connection, encounter_id: int) -> bool: + def delete_encounter(conn: Any, encounter_id: int) -> bool: """Delete one encounter. Returns True if a row was removed, False if not found. ON DELETE CASCADE handles combatants / damage_types / attack_types / ingest_log.""" - with conn: - cur = conn.execute(_SQL["delete_encounter"], (encounter_id,)) + cur = conn.execute(_SQL["delete_encounter"], (encounter_id,)) + conn.commit() return cur.rowcount > 0 @staticmethod - def soft_delete_encounter( - conn: sqlite3.Connection, encounter_id: int, hidden_at: int, hidden_by: str | None = None - ) -> bool: + def soft_delete_encounter(conn: Any, encounter_id: int, hidden_at: int, hidden_by: str | None = None) -> bool: """Hide an encounter from the parses list without removing it, so any leaderboard entry sourced from it survives and its link still opens. ``hidden_by`` records the actor's discord id for the admin view. Only acts on a currently-visible row; returns True if it flipped one.""" - with conn: - cur = conn.execute( - _SQL["soft_delete_encounter"], - (hidden_at, hidden_by, encounter_id), - ) + cur = conn.execute( + _SQL["soft_delete_encounter"], + (hidden_at, hidden_by, encounter_id), + ) + conn.commit() return cur.rowcount > 0 @staticmethod - def unhide_encounter(conn: sqlite3.Connection, encounter_id: int) -> bool: + def unhide_encounter(conn: Any, encounter_id: int) -> bool: """Clear a soft-delete marker so a previously-hidden parse becomes visible again (used when its encounter is re-uploaded). Returns True if a hidden row was un-hidden.""" - with conn: - cur = conn.execute( - _SQL["unhide_encounter"], - (encounter_id,), - ) + cur = conn.execute( + _SQL["unhide_encounter"], + (encounter_id,), + ) + conn.commit() return cur.rowcount > 0 @staticmethod - def set_encounter_guild_name(conn: sqlite3.Connection, encounter_id: int, guild_name: str | None) -> bool: + def set_encounter_guild_name(conn: Any, encounter_id: int, guild_name: str | None) -> bool: """Set (or clear) the guild_name on an encounter row. Returns True if the row was updated.""" - with conn: - cur = conn.execute( - _SQL["set_encounter_guild_name"], - (guild_name, encounter_id), - ) + cur = conn.execute( + _SQL["set_encounter_guild_name"], + (guild_name, encounter_id), + ) + conn.commit() return cur.rowcount > 0 @staticmethod - def get_combatants_for_encounter(conn: sqlite3.Connection, encounter_id: int) -> list[dict]: - conn.row_factory = sqlite3.Row + def get_combatants_for_encounter(conn: Any, encounter_id: int) -> list[dict]: rows = conn.execute( _SQL["get_combatants_for_encounter"], (encounter_id,), @@ -541,109 +387,118 @@ def get_combatants_for_encounter(conn: sqlite3.Connection, encounter_id: int) -> return [dict(r) for r in rows] @staticmethod - def get_combatants_for_encounters(conn: sqlite3.Connection, encounter_ids: list[int]) -> dict[int, list[dict]]: - """Batched :meth:`get_combatants_for_encounter` — one query per 500 - encounters instead of one per encounter (the rankings rebuild used - to N+1 this over the whole leaderboard). Rows keep the per-encounter - damage-DESC order.""" - conn.row_factory = sqlite3.Row + def get_combatants_for_encounters(conn: Any, encounter_ids: list[int]) -> dict[int, list[dict]]: + """Batched :meth:`get_combatants_for_encounter` — ONE ``= ANY`` query + (the SQLite 500-id IN-list chunking is gone). Rows keep the + per-encounter damage-DESC order.""" out: dict[int, list[dict]] = {eid: [] for eid in encounter_ids} - chunk_size = 500 - for i in range(0, len(encounter_ids), chunk_size): - chunk = encounter_ids[i : i + chunk_size] - rows = conn.execute( - _SQL["get_combatants_for_encounters"].format(placeholders=",".join("?" * len(chunk))), - chunk, - ).fetchall() - for r in rows: - out[r["encounter_id"]].append(dict(r)) + if not encounter_ids: + return out + rows = conn.execute(_SQL["get_combatants_for_encounters"], (list(encounter_ids),)).fetchall() + for r in rows: + out[r["encounter_id"]].append(dict(r)) return out @staticmethod def get_top_attacks_for_combatant( - conn: sqlite3.Connection, + conn: Any, combatant_id: int, limit: int = 10, ) -> list[dict]: """Top damage abilities (excludes heals and the swing_type=100 rollup).""" - conn.row_factory = sqlite3.Row - placeholders = ",".join("?" * len(_DAMAGE_SWING_TYPES)) rows = conn.execute( - _SQL["get_top_attacks_by_swing_type"].format(placeholders=placeholders), - (combatant_id, *_DAMAGE_SWING_TYPES, limit), + _SQL["get_top_attacks_by_swing_type"], + (combatant_id, _DAMAGE_SWING_TYPES, limit), ).fetchall() return [dict(r) for r in rows] @staticmethod def get_top_heals_for_combatant( - conn: sqlite3.Connection, + conn: Any, combatant_id: int, limit: int = 10, ) -> list[dict]: """Top heal abilities (swing_type=3). `damage` column = amount healed; `resist` column distinguishes regular 'Hitpoints' heals from 'Absorption' wards.""" - conn.row_factory = sqlite3.Row - placeholders = ",".join("?" * len(_HEAL_SWING_TYPES)) rows = conn.execute( - _SQL["get_top_attacks_by_swing_type"].format(placeholders=placeholders), - (combatant_id, *_HEAL_SWING_TYPES, limit), + _SQL["get_top_attacks_by_swing_type"], + (combatant_id, _HEAL_SWING_TYPES, limit), ).fetchall() return [dict(r) for r in rows] @staticmethod def get_top_cures_for_combatant( - conn: sqlite3.Connection, + conn: Any, combatant_id: int, limit: int = 10, ) -> list[dict]: """Cure events (swing_type=20). The `damage` column is the count of detrimental effects removed; `hits` is how many times the cure was cast.""" - conn.row_factory = sqlite3.Row - placeholders = ",".join("?" * len(_CURE_SWING_TYPES)) rows = conn.execute( - _SQL["get_top_cures"].format(placeholders=placeholders), - (combatant_id, *_CURE_SWING_TYPES, limit), + _SQL["get_top_cures"], + (combatant_id, _CURE_SWING_TYPES, limit), ).fetchall() return [dict(r) for r in rows] @staticmethod def get_top_threats_for_combatant( - conn: sqlite3.Connection, + conn: Any, combatant_id: int, limit: int = 10, ) -> list[dict]: """Threat / buff-proc rows (swing_type=100, type != 'All'). For threat procs the `damage` column is the threat-increase value; `hits` is proc count.""" - conn.row_factory = sqlite3.Row - placeholders = ",".join("?" * len(_THREAT_SWING_TYPES)) rows = conn.execute( - _SQL["get_top_threats"].format(placeholders=placeholders), - (combatant_id, *_THREAT_SWING_TYPES, limit), + _SQL["get_top_threats"], + (combatant_id, _THREAT_SWING_TYPES, limit), ).fetchall() return [dict(r) for r in rows] @staticmethod def get_damage_types_for_combatant( - conn: sqlite3.Connection, + conn: Any, combatant_id: int, ) -> list[dict]: """All damage_types rows for a combatant, sorted by damage DESC.""" - conn.row_factory = sqlite3.Row rows = conn.execute( _SQL["get_damage_types_for_combatant"], (combatant_id,), ).fetchall() return [dict(r) for r in rows] + # --------------------------------------------------------------------------- + # Tiered detail retention (cleanup sweep) + # --------------------------------------------------------------------------- + + @staticmethod + def select_detail_prune_candidates(conn: Any, *, older_than: int, limit: int = 500) -> list[dict]: + """Encounters older than ``older_than`` whose breakdown rows are still + present (detail_pruned_at IS NULL), oldest first. The sweep classifies + each row's zone into its retention tier and prunes the overdue subset.""" + rows = conn.execute(_SQL["select_detail_prune_candidates"], (older_than, limit)).fetchall() + return [dict(r) for r in rows] + + @staticmethod + def prune_encounter_detail(conn: Any, encounter_ids: list[int], *, pruned_at: int) -> None: + """Drop attack_types + damage_types for the given encounters and stamp + detail_pruned_at (encounters + combatants stay forever). One + transaction; caller commits.""" + if not encounter_ids: + return + ids = list(encounter_ids) + conn.execute(_SQL["prune_detail_attack_types"], (ids,)) + conn.execute(_SQL["prune_detail_damage_types"], (ids,)) + conn.execute(_SQL["mark_detail_pruned"], (pruned_at, ids)) + # --------------------------------------------------------------------------- # client_warnings (soft warnings on otherwise-successful uploads) # --------------------------------------------------------------------------- @staticmethod def set_encounter_client_warnings( - conn: sqlite3.Connection, + conn: Any, encounter_id: int, warnings_json: str | None, ) -> None: @@ -670,7 +525,7 @@ def set_encounter_client_warnings( @staticmethod def insert_tamper_report( - conn: sqlite3.Connection, + conn: Any, *, world: str, act_encid: str, @@ -719,11 +574,11 @@ def insert_tamper_report( ), ) row = cur.fetchone() - return int(row[0]) if row else 0 + return int(row["id"]) if row else 0 @staticmethod def list_tamper_reports( - conn: sqlite3.Connection, + conn: Any, *, world: str | None = None, reason: str | None = None, @@ -745,10 +600,10 @@ def list_tamper_reports( clauses: list[str] = [] params: list = [] if world is not None: - clauses.append("world = ?") + clauses.append("world = %s") params.append(world) if reason is not None: - clauses.append("reason = ?") + clauses.append("reason = %s") params.append(reason) if status == "pending": clauses.append("acknowledged_at IS NULL") @@ -759,13 +614,12 @@ def list_tamper_reports( else: raise ValueError(f"unknown status {status!r}") where = ("WHERE " + " AND ".join(clauses)) if clauses else "" - conn.row_factory = sqlite3.Row sql = _SQL["list_tamper_reports"].format(where=where) return [dict(r) for r in conn.execute(sql, [*params, limit]).fetchall()] @staticmethod def acknowledge_tamper_report( - conn: sqlite3.Connection, + conn: Any, report_id: int, *, acknowledged_at: int, @@ -778,16 +632,16 @@ def acknowledge_tamper_report( wants to revisit, they can read the row via the ``status="ack"`` or ``status="all"`` listing. """ - with conn: - cur = conn.execute( - _SQL["acknowledge_tamper_report"], - (acknowledged_at, acknowledged_by, report_id), - ) + cur = conn.execute( + _SQL["acknowledge_tamper_report"], + (acknowledged_at, acknowledged_by, report_id), + ) + conn.commit() return cur.rowcount > 0 @staticmethod def acknowledge_tamper_reports( - conn: sqlite3.Connection, + conn: Any, report_ids: list[int], *, acknowledged_at: int, @@ -797,17 +651,16 @@ def acknowledge_tamper_reports( and unknown ids are silently skipped); returns the flipped count.""" if not report_ids: return 0 - placeholders = ",".join("?" * len(report_ids)) - with conn: - cur = conn.execute( - _SQL["acknowledge_tamper_reports_bulk"].format(placeholders=placeholders), - [acknowledged_at, acknowledged_by, *report_ids], - ) + cur = conn.execute( + _SQL["acknowledge_tamper_reports_bulk"], + (acknowledged_at, acknowledged_by, list(report_ids)), + ) + conn.commit() return cur.rowcount @staticmethod def acknowledge_all_pending_tamper_reports( - conn: sqlite3.Connection, + conn: Any, world: str, *, acknowledged_at: int, @@ -816,25 +669,25 @@ def acknowledge_all_pending_tamper_reports( """Ack every pending report for a world in one statement — the spam cleanup path (a single hammering uploader can create far more rows than the 500-id batch endpoint can clear). Returns the flipped count.""" - with conn: - cur = conn.execute( - _SQL["acknowledge_all_pending_tamper_reports"], - (acknowledged_at, acknowledged_by, world), - ) + cur = conn.execute( + _SQL["acknowledge_all_pending_tamper_reports"], + (acknowledged_at, acknowledged_by, world), + ) + conn.commit() return cur.rowcount @staticmethod - def delete_acknowledged_tamper_reports(conn: sqlite3.Connection, world: str) -> int: + def delete_acknowledged_tamper_reports(conn: Any, world: str) -> int: """Hard-delete already-acknowledged reports for a world to reclaim space after a spam flood. Reviewed rows only — pending ones are never touched. Returns the deleted count.""" - with conn: - cur = conn.execute(_SQL["delete_acknowledged_tamper_reports"], (world,)) + cur = conn.execute(_SQL["delete_acknowledged_tamper_reports"], (world,)) + conn.commit() return cur.rowcount @staticmethod def count_pending_tamper_reports( - conn: sqlite3.Connection, + conn: Any, world: str | None = None, ) -> int: """Cheap count of unack'd reports — used by the admin panel badge so @@ -846,7 +699,7 @@ def count_pending_tamper_reports( _SQL["count_pending_tamper_reports_for_world"], (world,), ).fetchone() - return int(row[0]) if row else 0 + return int(row["n"]) if row else 0 # The shared default instance — every runtime consumer goes through this. diff --git a/backend/server/parses/db.sql b/backend/server/parses/db.sql index 6633a8fd..89674d5b 100644 --- a/backend/server/parses/db.sql +++ b/backend/server/parses/db.sql @@ -1,322 +1,8 @@ --- SQL for backend/server/parses/db.py — parses DB schema + DML. --- Schema is read in init_db; migration rebuilds use a Python .replace() on --- the schema_* string to swap IF NOT EXISTS off and the table name to the --- _new sentinel — clean idempotency without a second copy of the CREATE. +-- SQL for backend/server/parses/db.py (psycopg, parses schema). +-- Schema DDL lives in db/migrations/0002_parses.sql; this file is DML only. +-- The four insert blocks use %(name)s placeholders matched against the +-- models' as_db_params() dicts — change column ↔ field mappings there. --- --------------------------------------------------------------------------- --- Schema --- --------------------------------------------------------------------------- - --- :name schema_encounters -CREATE TABLE IF NOT EXISTS encounters ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL DEFAULT 'Varsoon', - act_encid TEXT NOT NULL, - title TEXT NOT NULL, - zone TEXT, - started_at INTEGER NOT NULL, -- unix seconds, UTC - ended_at INTEGER NOT NULL, - duration_s INTEGER NOT NULL, - total_damage INTEGER NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - kills INTEGER NOT NULL DEFAULT 0, - deaths INTEGER NOT NULL DEFAULT 0, - -- ACT's GetEncounterSuccessLevel(): 0=unknown, 1=win, 2=loss, 3=mixed. - -- Used by /parses to colour the encounter title green/red. - success_level INTEGER NOT NULL DEFAULT 0, - source_dsn TEXT NOT NULL, - uploaded_by TEXT NOT NULL DEFAULT 'local', - guild_name TEXT, - ingested_at INTEGER NOT NULL, - -- Soft-delete marker (unix seconds). NULL = visible. Set when a boss-kill - -- parse is "deleted" so the leaderboard entry + its link survive while the - -- row is hidden from the /parses list. Hard purge removes the row entirely. - hidden_at INTEGER, - -- Discord id of whoever hid the row (admin/officer/uploader) — the - -- admin sanitize view shows it; cleared on unhide. - hidden_by TEXT, - UNIQUE (world, act_encid) -); - --- :name schema_combatants -CREATE TABLE IF NOT EXISTS combatants ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - encounter_id INTEGER NOT NULL, - name TEXT NOT NULL, - ally INTEGER NOT NULL DEFAULT 0, -- 0/1 (ACT's 'T'/'F') - started_at INTEGER NOT NULL DEFAULT 0, - ended_at INTEGER NOT NULL DEFAULT 0, - duration_s INTEGER NOT NULL DEFAULT 0, - damage INTEGER NOT NULL DEFAULT 0, - damage_perc REAL NOT NULL DEFAULT 0, - kills INTEGER NOT NULL DEFAULT 0, - healed INTEGER NOT NULL DEFAULT 0, - healed_perc REAL NOT NULL DEFAULT 0, - crit_heals INTEGER NOT NULL DEFAULT 0, - heals INTEGER NOT NULL DEFAULT 0, - cure_dispels INTEGER NOT NULL DEFAULT 0, - power_drain INTEGER NOT NULL DEFAULT 0, - power_replenish INTEGER NOT NULL DEFAULT 0, - dps REAL NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - enchps REAL NOT NULL DEFAULT 0, - hits INTEGER NOT NULL DEFAULT 0, - crit_hits INTEGER NOT NULL DEFAULT 0, - blocked INTEGER NOT NULL DEFAULT 0, - misses INTEGER NOT NULL DEFAULT 0, - swings INTEGER NOT NULL DEFAULT 0, - heals_taken INTEGER NOT NULL DEFAULT 0, - damage_taken INTEGER NOT NULL DEFAULT 0, - deaths INTEGER NOT NULL DEFAULT 0, - to_hit REAL NOT NULL DEFAULT 0, - crit_dam_perc REAL NOT NULL DEFAULT 0, - crit_heal_perc REAL NOT NULL DEFAULT 0, - crit_types TEXT, - threat_str TEXT, - threat_delta INTEGER NOT NULL DEFAULT 0, - -- Identity snapshot frozen at ingest (resolved via character_cache). - -- NULL for pets/NPCs and players we couldn't resolve at upload time. - level INTEGER, - guild_name TEXT, - cls TEXT, - ilvl REAL, - FOREIGN KEY (encounter_id) REFERENCES encounters(id) ON DELETE CASCADE, - UNIQUE (encounter_id, name) -); - --- :name schema_damage_types -CREATE TABLE IF NOT EXISTS damage_types ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - combatant_id INTEGER NOT NULL, - grouping_label TEXT, - damage_type TEXT NOT NULL, - started_at INTEGER NOT NULL DEFAULT 0, - ended_at INTEGER NOT NULL DEFAULT 0, - duration_s INTEGER NOT NULL DEFAULT 0, - damage INTEGER NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - char_dps REAL NOT NULL DEFAULT 0, - dps REAL NOT NULL DEFAULT 0, - average REAL NOT NULL DEFAULT 0, - median INTEGER NOT NULL DEFAULT 0, - min_hit INTEGER NOT NULL DEFAULT 0, - max_hit INTEGER NOT NULL DEFAULT 0, - hits INTEGER NOT NULL DEFAULT 0, - crit_hits INTEGER NOT NULL DEFAULT 0, - blocked INTEGER NOT NULL DEFAULT 0, - misses INTEGER NOT NULL DEFAULT 0, - swings INTEGER NOT NULL DEFAULT 0, - to_hit REAL NOT NULL DEFAULT 0, - average_delay REAL NOT NULL DEFAULT 0, - crit_perc REAL NOT NULL DEFAULT 0, - crit_types TEXT, - FOREIGN KEY (combatant_id) REFERENCES combatants(id) ON DELETE CASCADE, - UNIQUE (combatant_id, damage_type) -); - --- :name schema_attack_types -CREATE TABLE IF NOT EXISTS attack_types ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - combatant_id INTEGER NOT NULL, - victim TEXT, - swing_type INTEGER NOT NULL DEFAULT 0, - attack_name TEXT NOT NULL, - started_at INTEGER NOT NULL DEFAULT 0, - ended_at INTEGER NOT NULL DEFAULT 0, - duration_s INTEGER NOT NULL DEFAULT 0, - damage INTEGER NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - char_dps REAL NOT NULL DEFAULT 0, - dps REAL NOT NULL DEFAULT 0, - average REAL NOT NULL DEFAULT 0, - median INTEGER NOT NULL DEFAULT 0, - min_hit INTEGER NOT NULL DEFAULT 0, - max_hit INTEGER NOT NULL DEFAULT 0, - resist TEXT, - hits INTEGER NOT NULL DEFAULT 0, - crit_hits INTEGER NOT NULL DEFAULT 0, - blocked INTEGER NOT NULL DEFAULT 0, - misses INTEGER NOT NULL DEFAULT 0, - swings INTEGER NOT NULL DEFAULT 0, - to_hit REAL NOT NULL DEFAULT 0, - average_delay REAL NOT NULL DEFAULT 0, - crit_perc REAL NOT NULL DEFAULT 0, - crit_types TEXT, - FOREIGN KEY (combatant_id) REFERENCES combatants(id) ON DELETE CASCADE, - UNIQUE (combatant_id, swing_type, attack_name) -); - --- :name schema_ingest_log -CREATE TABLE IF NOT EXISTS ingest_log ( - world TEXT NOT NULL DEFAULT 'Varsoon', - act_encid TEXT NOT NULL, - encounter_id INTEGER NOT NULL, - ingested_at INTEGER NOT NULL, - source_dsn TEXT NOT NULL, - PRIMARY KEY (world, act_encid), - FOREIGN KEY (encounter_id) REFERENCES encounters(id) ON DELETE CASCADE -); - --- :name schema_tamper_reports -CREATE TABLE IF NOT EXISTS tamper_reports ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - world TEXT NOT NULL DEFAULT 'Varsoon', - act_encid TEXT NOT NULL, - title TEXT NOT NULL, - zone TEXT, - started_at INTEGER NOT NULL, -- unix seconds, UTC - ended_at INTEGER NOT NULL, - duration_s INTEGER NOT NULL, - total_damage INTEGER NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - reason TEXT NOT NULL, - reported_at INTEGER NOT NULL, - uploader_logger_name TEXT NOT NULL DEFAULT '', - uploader_discord_id TEXT NOT NULL DEFAULT '', - uploader_discord_name TEXT NOT NULL DEFAULT '', - guild_name TEXT, - payload_json TEXT NOT NULL, - acknowledged_at INTEGER, - acknowledged_by TEXT -); - --- Multi-statement block — init_db runs via conn.executescript(). --- :name indexes_all -CREATE INDEX IF NOT EXISTS idx_encounters_started_desc ON encounters (started_at DESC); -CREATE INDEX IF NOT EXISTS idx_encounters_zone ON encounters (zone); -CREATE INDEX IF NOT EXISTS idx_encounters_world ON encounters (world, started_at DESC); -CREATE INDEX IF NOT EXISTS idx_encounters_uploaded_by ON encounters (uploaded_by, started_at DESC); -CREATE INDEX IF NOT EXISTS idx_combatants_encounter ON combatants (encounter_id); -CREATE INDEX IF NOT EXISTS idx_combatants_name ON combatants (name); -CREATE INDEX IF NOT EXISTS idx_combatants_ally ON combatants (encounter_id, ally); -CREATE INDEX IF NOT EXISTS idx_damage_types_combatant ON damage_types (combatant_id); -CREATE INDEX IF NOT EXISTS idx_attack_types_combatant ON attack_types (combatant_id); -CREATE INDEX IF NOT EXISTS idx_attack_types_damage_desc ON attack_types (combatant_id, damage DESC); -CREATE INDEX IF NOT EXISTS idx_combatants_encounter_is_player ON combatants (encounter_id, is_player); -CREATE INDEX IF NOT EXISTS idx_combatants_rankings_cover ON combatants - (encounter_id, ally, is_player, name, cls, level, ilvl, guild_name, encdps, enchps, damage, healed, deaths); -CREATE INDEX IF NOT EXISTS idx_tamper_reports_unack ON tamper_reports (reported_at DESC) WHERE acknowledged_at IS NULL; -CREATE INDEX IF NOT EXISTS idx_tamper_reports_reporter ON tamper_reports (uploader_discord_id, reported_at DESC); -CREATE INDEX IF NOT EXISTS idx_tamper_reports_world_reported ON tamper_reports (world, reported_at DESC); - --- --------------------------------------------------------------------------- --- PRAGMAs (connection setup + introspection + migration toggles) --- --------------------------------------------------------------------------- - --- :name pragma_foreign_keys_off -PRAGMA foreign_keys = OFF; - --- :name pragma_legacy_alter_table_on -PRAGMA legacy_alter_table = ON; - --- :name pragma_foreign_keys_on -PRAGMA foreign_keys = ON; - --- :name pragma_legacy_alter_table_off -PRAGMA legacy_alter_table = OFF; - --- :name pragma_table_info_encounters -PRAGMA table_info(encounters); - --- :name pragma_table_info_ingest_log -PRAGMA table_info(ingest_log); - --- {idx_name} is interpolated by Python (no parameter binding for PRAGMA args). --- :name pragma_index_info -PRAGMA index_info({idx_name}); - --- --------------------------------------------------------------------------- --- Idempotent ALTER migrations (looped by init_db; each may already be applied) --- --------------------------------------------------------------------------- - --- :name alter_encounters_add_uploaded_by -ALTER TABLE encounters ADD COLUMN uploaded_by TEXT NOT NULL DEFAULT 'local'; - --- :name alter_encounters_add_guild_name -ALTER TABLE encounters ADD COLUMN guild_name TEXT; - --- :name alter_encounters_add_success_level -ALTER TABLE encounters ADD COLUMN success_level INTEGER NOT NULL DEFAULT 0; - --- :name alter_combatants_add_level -ALTER TABLE combatants ADD COLUMN level INTEGER; - --- :name alter_combatants_add_guild_name -ALTER TABLE combatants ADD COLUMN guild_name TEXT; - --- :name alter_combatants_add_cls -ALTER TABLE combatants ADD COLUMN cls TEXT; - --- :name alter_combatants_add_ilvl -ALTER TABLE combatants ADD COLUMN ilvl REAL; - --- :name alter_encounters_add_hidden_at -ALTER TABLE encounters ADD COLUMN hidden_at INTEGER; - --- :name alter_encounters_add_hidden_by -ALTER TABLE encounters ADD COLUMN hidden_by TEXT; - --- :name alter_combatants_add_is_player -ALTER TABLE combatants ADD COLUMN is_player INTEGER DEFAULT NULL; - --- :name alter_encounters_add_client_warnings -ALTER TABLE encounters ADD COLUMN client_warnings TEXT; - --- --------------------------------------------------------------------------- --- Migration helpers (table rebuilds — used by _migrate_* functions) --- --------------------------------------------------------------------------- - --- :name migrate_check_attack_types_indexes -SELECT name FROM sqlite_master -WHERE type='index' AND tbl_name='attack_types' -AND name LIKE 'sqlite_autoindex_%'; - --- :name migrate_attack_types_insert_into_new -INSERT INTO attack_types_new SELECT * FROM attack_types; - --- :name migrate_attack_types_drop_old -DROP TABLE attack_types; - --- :name migrate_attack_types_rename -ALTER TABLE attack_types_new RENAME TO attack_types; - --- :name migrate_encounters_rename_old -ALTER TABLE encounters RENAME TO encounters_old; - --- :name migrate_encounters_copy_from_old -INSERT INTO encounters ( - id, world, act_encid, title, zone, - started_at, ended_at, duration_s, - total_damage, encdps, kills, deaths, success_level, - source_dsn, uploaded_by, guild_name, ingested_at, hidden_at -) -SELECT - id, 'Varsoon', act_encid, title, zone, - started_at, ended_at, duration_s, - total_damage, encdps, kills, deaths, success_level, - source_dsn, uploaded_by, guild_name, ingested_at, hidden_at -FROM encounters_old; - --- :name migrate_encounters_drop_old -DROP TABLE encounters_old; - --- :name migrate_ingest_log_rename_old -ALTER TABLE ingest_log RENAME TO ingest_log_old; - --- :name migrate_ingest_log_copy_from_old -INSERT INTO ingest_log (world, act_encid, encounter_id, ingested_at, source_dsn) -SELECT 'Varsoon', act_encid, encounter_id, ingested_at, source_dsn -FROM ingest_log_old; - --- :name migrate_ingest_log_drop_old -DROP TABLE ingest_log_old; - --- --------------------------------------------------------------------------- --- Insert helpers --- --------------------------------------------------------------------------- - --- Named :param placeholders are matched against Encounter.as_db_params() --- — change column ↔ field mappings there, not by re-ordering values here. -- :name insert_encounter INSERT INTO encounters ( world, act_encid, title, zone, @@ -324,14 +10,13 @@ INSERT INTO encounters ( total_damage, encdps, kills, deaths, success_level, source_dsn, uploaded_by, guild_name, ingested_at ) VALUES ( - :world, :act_encid, :title, :zone, - :started_at, :ended_at, :duration_s, - :total_damage, :encdps, :kills, :deaths, :success_level, - :source_dsn, :uploaded_by, :guild_name, :ingested_at + %(world)s, %(act_encid)s, %(title)s, %(zone)s, + %(started_at)s, %(ended_at)s, %(duration_s)s, + %(total_damage)s, %(encdps)s, %(kills)s, %(deaths)s, %(success_level)s, + %(source_dsn)s, %(uploaded_by)s, %(guild_name)s, %(ingested_at)s ) RETURNING id; --- Named :param placeholders matched against Combatant.as_db_params(). -- :name insert_combatant INSERT INTO combatants ( encounter_id, name, ally, @@ -346,31 +31,30 @@ INSERT INTO combatants ( threat_str, threat_delta, level, guild_name, cls, ilvl ) VALUES ( - :encounter_id, :name, :ally, - :started_at, :ended_at, :duration_s, - :damage, :damage_perc, :kills, - :healed, :healed_perc, :crit_heals, :heals, :cure_dispels, - :power_drain, :power_replenish, - :dps, :encdps, :enchps, - :hits, :crit_hits, :blocked, :misses, :swings, - :heals_taken, :damage_taken, :deaths, - :to_hit, :crit_dam_perc, :crit_heal_perc, :crit_types, - :threat_str, :threat_delta, - :level, :guild_name, :cls, :ilvl + %(encounter_id)s, %(name)s, %(ally)s, + %(started_at)s, %(ended_at)s, %(duration_s)s, + %(damage)s, %(damage_perc)s, %(kills)s, + %(healed)s, %(healed_perc)s, %(crit_heals)s, %(heals)s, %(cure_dispels)s, + %(power_drain)s, %(power_replenish)s, + %(dps)s, %(encdps)s, %(enchps)s, + %(hits)s, %(crit_hits)s, %(blocked)s, %(misses)s, %(swings)s, + %(heals_taken)s, %(damage_taken)s, %(deaths)s, + %(to_hit)s, %(crit_dam_perc)s, %(crit_heal_perc)s, %(crit_types)s, + %(threat_str)s, %(threat_delta)s, + %(level)s, %(guild_name)s, %(cls)s, %(ilvl)s ) RETURNING id; -- :name update_combatant_snapshot -UPDATE combatants SET level = ?, guild_name = ?, cls = ?, ilvl = ? -WHERE encounter_id = ? AND name = ?; +UPDATE combatants SET level = %s, guild_name = %s, cls = %s, ilvl = %s +WHERE encounter_id = %s AND name = %s; -- :name update_combatant_is_player -UPDATE combatants SET is_player = ? WHERE id = ?; +UPDATE combatants SET is_player = %s WHERE id = %s; -- :name invalidate_is_player_cache UPDATE combatants SET is_player = NULL; --- Named :param placeholders matched against DamageType.as_db_params(). -- :name insert_damage_type INSERT INTO damage_types ( combatant_id, grouping_label, damage_type, @@ -380,15 +64,14 @@ INSERT INTO damage_types ( hits, crit_hits, blocked, misses, swings, to_hit, average_delay, crit_perc, crit_types ) VALUES ( - :combatant_id, :grouping_label, :damage_type, - :started_at, :ended_at, :duration_s, - :damage, :encdps, :char_dps, :dps, - :average, :median, :min_hit, :max_hit, - :hits, :crit_hits, :blocked, :misses, :swings, - :to_hit, :average_delay, :crit_perc, :crit_types + %(combatant_id)s, %(grouping_label)s, %(damage_type)s, + %(started_at)s, %(ended_at)s, %(duration_s)s, + %(damage)s, %(encdps)s, %(char_dps)s, %(dps)s, + %(average)s, %(median)s, %(min_hit)s, %(max_hit)s, + %(hits)s, %(crit_hits)s, %(blocked)s, %(misses)s, %(swings)s, + %(to_hit)s, %(average_delay)s, %(crit_perc)s, %(crit_types)s ); --- Named :param placeholders matched against AttackType.as_db_params(). -- :name insert_attack_type INSERT INTO attack_types ( combatant_id, victim, swing_type, attack_name, @@ -398,103 +81,103 @@ INSERT INTO attack_types ( hits, crit_hits, blocked, misses, swings, to_hit, average_delay, crit_perc, crit_types ) VALUES ( - :combatant_id, :victim, :swing_type, :attack_name, - :started_at, :ended_at, :duration_s, - :damage, :encdps, :char_dps, :dps, - :average, :median, :min_hit, :max_hit, :resist, - :hits, :crit_hits, :blocked, :misses, :swings, - :to_hit, :average_delay, :crit_perc, :crit_types + %(combatant_id)s, %(victim)s, %(swing_type)s, %(attack_name)s, + %(started_at)s, %(ended_at)s, %(duration_s)s, + %(damage)s, %(encdps)s, %(char_dps)s, %(dps)s, + %(average)s, %(median)s, %(min_hit)s, %(max_hit)s, %(resist)s, + %(hits)s, %(crit_hits)s, %(blocked)s, %(misses)s, %(swings)s, + %(to_hit)s, %(average_delay)s, %(crit_perc)s, %(crit_types)s ); -- :name mark_ingested INSERT INTO ingest_log (world, act_encid, encounter_id, ingested_at, source_dsn) -VALUES (?, ?, ?, ?, ?); +VALUES (%s, %s, %s, %s, %s); -- --------------------------------------------------------------------------- -- Lookup helpers -- --------------------------------------------------------------------------- -- :name check_is_ingested -SELECT 1 FROM ingest_log WHERE world = ? AND act_encid = ? LIMIT 1; +SELECT 1 FROM ingest_log WHERE world = %s AND act_encid = %s LIMIT 1; -- :name find_encounter_by_act_encid -SELECT * FROM encounters WHERE world = ? AND act_encid = ? LIMIT 1; +SELECT * FROM encounters WHERE world = %s AND act_encid = %s LIMIT 1; -- :name recent_encounters_by_zone SELECT * FROM encounters -WHERE world = ? AND zone = ? +WHERE world = %s AND zone = %s ORDER BY started_at DESC -LIMIT ?; +LIMIT %s; -- :name recent_encounters_all -SELECT * FROM encounters WHERE world = ? ORDER BY started_at DESC LIMIT ?; +SELECT * FROM encounters WHERE world = %s ORDER BY started_at DESC LIMIT %s; --- :name list_encounters_for_admin -- {where} = "WHERE …" composed in Python; embeds a correlated subquery for --- the player_count column. +-- the player_count column. (Comment kept OUTSIDE the block — the composed +-- fragment contains %s and psycopg counts placeholders inside comments.) +-- :name list_encounters_for_admin SELECT e.id, e.title, e.zone, e.guild_name, e.uploaded_by, e.started_at, e.duration_s, e.success_level, e.hidden_at, e.hidden_by, e.client_warnings, (SELECT COUNT(*) FROM combatants c WHERE c.encounter_id = e.id AND c.ally = 1 AND c.name != '' AND c.name != 'Unknown' - AND instr(c.name, ' ') = 0) AS player_count + AND strpos(c.name, ' ') = 0) AS player_count FROM encounters e {where} ORDER BY e.started_at DESC -LIMIT ?; +LIMIT %s; -- :name delete_encounter -DELETE FROM encounters WHERE id = ?; +DELETE FROM encounters WHERE id = %s; -- :name soft_delete_encounter -UPDATE encounters SET hidden_at = ?, hidden_by = ? WHERE id = ? AND hidden_at IS NULL; +UPDATE encounters SET hidden_at = %s, hidden_by = %s WHERE id = %s AND hidden_at IS NULL; -- :name unhide_encounter -UPDATE encounters SET hidden_at = NULL, hidden_by = NULL WHERE id = ? AND hidden_at IS NOT NULL; +UPDATE encounters SET hidden_at = NULL, hidden_by = NULL WHERE id = %s AND hidden_at IS NOT NULL; -- :name set_encounter_guild_name -UPDATE encounters SET guild_name = ? WHERE id = ?; +UPDATE encounters SET guild_name = %s WHERE id = %s; -- :name get_combatants_for_encounter -SELECT * FROM combatants WHERE encounter_id = ? ORDER BY damage DESC; - +SELECT * FROM combatants WHERE encounter_id = %s ORDER BY damage DESC; + +-- Batched form of get_combatants_for_encounter: the rankings rebuild fetches +-- every primary kill's combatants in ONE query (= ANY replaced the SQLite +-- 500-id IN-list chunking). Narrowed to exactly the columns the rankings / +-- export / character-rankings pipelines read, and shaped to be COVERED by +-- idx_combatants_rankings_cover: on the Railway network volume, SELECT * +-- random-paged ~150k wide rows and turned the Wuoshi kills rebuild into a +-- 14-minute grind (combatants=837s, 2026-09-12); the covering index turns it +-- into dense index-only scans. -- :name get_combatants_for_encounters --- {placeholders} = comma-joined "?,?,..." for the IN list. Batched form of --- get_combatants_for_encounter: the rankings rebuild fetches every primary --- kill's combatants in a handful of queries instead of one per kill. --- Narrowed to exactly the columns the rankings/export/character-rankings --- pipelines read, and shaped to be COVERED by idx_combatants_rankings_cover: --- on the Railway network volume, SELECT * random-paged ~150k wide rows and --- turned the Wuoshi kills rebuild into a 14-minute grind (combatants=837s, --- 2026-09-12); the covering index turns it into dense index-only scans. SELECT encounter_id, name, ally, is_player, cls, level, ilvl, guild_name, encdps, enchps, damage, healed, deaths -FROM combatants WHERE encounter_id IN ({placeholders}) ORDER BY encounter_id, damage DESC; +FROM combatants WHERE encounter_id = ANY(%s) ORDER BY encounter_id, damage DESC; -- :name get_top_attacks_by_swing_type --- {placeholders} = comma-joined "?,?,..." for the IN list. SELECT * FROM attack_types -WHERE combatant_id = ? AND swing_type IN ({placeholders}) +WHERE combatant_id = %s AND swing_type = ANY(%s) ORDER BY damage DESC -LIMIT ?; +LIMIT %s; -- :name get_top_cures SELECT * FROM attack_types -WHERE combatant_id = ? AND swing_type IN ({placeholders}) +WHERE combatant_id = %s AND swing_type = ANY(%s) ORDER BY hits DESC, damage DESC -LIMIT ?; +LIMIT %s; -- :name get_top_threats SELECT * FROM attack_types -WHERE combatant_id = ? - AND swing_type IN ({placeholders}) +WHERE combatant_id = %s + AND swing_type = ANY(%s) AND attack_name <> 'All' ORDER BY damage DESC -LIMIT ?; +LIMIT %s; -- :name get_damage_types_for_combatant SELECT * FROM damage_types -WHERE combatant_id = ? +WHERE combatant_id = %s ORDER BY damage DESC; -- --------------------------------------------------------------------------- @@ -502,7 +185,7 @@ ORDER BY damage DESC; -- --------------------------------------------------------------------------- -- :name set_encounter_client_warnings -UPDATE encounters SET client_warnings = ? WHERE id = ?; +UPDATE encounters SET client_warnings = %s WHERE id = %s; -- :name insert_tamper_report INSERT INTO tamper_reports ( @@ -513,17 +196,18 @@ INSERT INTO tamper_reports ( uploader_logger_name, uploader_discord_id, uploader_discord_name, guild_name, payload_json ) VALUES ( - ?, ?, ?, ?, - ?, ?, ?, - ?, ?, - ?, ?, - ?, ?, ?, - ?, ? + %s, %s, %s, %s, + %s, %s, %s, + %s, %s, + %s, %s, + %s, %s, %s, + %s, %s ) RETURNING id; +-- {where} composed in Python (filters: world / reason / pending|ack|all). +-- Comment kept OUTSIDE the block — see list_encounters_for_admin. -- :name list_tamper_reports --- {where} composed in Python (filters: world / reason / pending|ack|all) SELECT id, world, act_encid, title, zone, started_at, ended_at, duration_s, total_damage, encdps, @@ -534,34 +218,58 @@ SELECT id, world, act_encid, title, zone, FROM tamper_reports {where} ORDER BY reported_at DESC -LIMIT ?; +LIMIT %s; -- :name acknowledge_tamper_report UPDATE tamper_reports - SET acknowledged_at = ?, acknowledged_by = ? - WHERE id = ? AND acknowledged_at IS NULL; + SET acknowledged_at = %s, acknowledged_by = %s + WHERE id = %s AND acknowledged_at IS NULL; -- :name acknowledge_tamper_reports_bulk --- {placeholders} = comma-joined "?,?,..." UPDATE tamper_reports - SET acknowledged_at = ?, acknowledged_by = ? - WHERE id IN ({placeholders}) AND acknowledged_at IS NULL; + SET acknowledged_at = %s, acknowledged_by = %s + WHERE id = ANY(%s) AND acknowledged_at IS NULL; -- :name count_pending_tamper_reports -SELECT COUNT(*) FROM tamper_reports WHERE acknowledged_at IS NULL; +SELECT COUNT(*) AS n FROM tamper_reports WHERE acknowledged_at IS NULL; -- :name count_pending_tamper_reports_for_world -SELECT COUNT(*) FROM tamper_reports WHERE world = ? AND acknowledged_at IS NULL; +SELECT COUNT(*) AS n FROM tamper_reports WHERE world = %s AND acknowledged_at IS NULL; --- :name acknowledge_all_pending_tamper_reports -- Ack EVERY pending report for a world in one statement — the spam-flood -- escape hatch (a hammering uploader can create more than the 500-id batch -- endpoint can clear in a sane number of round-trips). +-- :name acknowledge_all_pending_tamper_reports UPDATE tamper_reports - SET acknowledged_at = ?, acknowledged_by = ? - WHERE world = ? AND acknowledged_at IS NULL; + SET acknowledged_at = %s, acknowledged_by = %s + WHERE world = %s AND acknowledged_at IS NULL; --- :name delete_acknowledged_tamper_reports -- Hard-delete already-reviewed reports for a world to reclaim space. Pending -- rows are never touched. -DELETE FROM tamper_reports WHERE world = ? AND acknowledged_at IS NOT NULL; +-- :name delete_acknowledged_tamper_reports +DELETE FROM tamper_reports WHERE world = %s AND acknowledged_at IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- Tiered detail retention (cleanup sweep; user decision 2026-10-04) +-- --------------------------------------------------------------------------- + +-- Candidate encounters whose breakdown rows may be due for pruning: older +-- than the LONGEST tier, detail not yet pruned, not soft-deleted. The sweep +-- classifies each zone in Python (curated raid 30d / group-instance 14d / +-- other 7d) and prunes the subset whose tier cutoff has actually passed. +-- :name select_detail_prune_candidates +SELECT id, zone, started_at FROM encounters +WHERE detail_pruned_at IS NULL AND started_at < %s +ORDER BY started_at +LIMIT %s; + +-- :name prune_detail_attack_types +DELETE FROM attack_types +WHERE combatant_id IN (SELECT id FROM combatants WHERE encounter_id = ANY(%s)); + +-- :name prune_detail_damage_types +DELETE FROM damage_types +WHERE combatant_id IN (SELECT id FROM combatants WHERE encounter_id = ANY(%s)); + +-- :name mark_detail_pruned +UPDATE encounters SET detail_pruned_at = %s WHERE id = ANY(%s); From 25c7f40152e6931b7452e3d854e7c5aed17bfb44 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 15:52:05 +0100 Subject: [PATCH 18/32] =?UTF-8?q?feat(pg):=20parses=20family=20runtime=20o?= =?UTF-8?q?n=20psycopg=20=E2=80=94=20routes,=20sweeps,=20erasure=20unified?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every parses consumer now rides the pooled schema-scoped proxy: parses package routes (list/ingest/delete/tamper/export), rankings' kills rebuild, zones progress stitches (zones.db side stays SQLite), admin sanitize/tamper panels, attendance reconstruct (COLLATE NOCASE -> lower() on the new index), bot parse_posts, dev seed scripts. path guards deleted; IN-list chunking -> ANY; instr->strpos; IFNULL-> COALESCE; dict rows throughout; PG requires the derived-table alias SQLite tolerated in list_encounters_recent. cleanup.py gains the tiered detail-retention sweep (user decision): curated raid 30d / curated group-instance 14d / other 7d via _classify_zone; attack_types+damage_types dropped, detail_pruned_at stamped, encounters+combatants forever. Erasure is now ONE transaction spanning users+parses schemas (search_path switch mid-txn). Metrics: parses counts via PG + a pg_schema_size_bytes gauge per family (the Supabase tier-gate measurement). PgConnProxy.close() rolls back implicit read transactions (silences psycopg_pool's warning). Parses test fixtures lease scratch schemas from 0002. Verified by a live store + tiered-retention smoke and the app lifespan boot on local PG17. Co-Authored-By: Claude Fable 5 --- backend/bot/cogs/parse_posts.py | 16 ++- backend/db_catalogue.py | 8 ++ backend/server/api/admin.py | 12 -- backend/server/api/attendance.py | 14 +-- backend/server/api/parses/delete.py | 10 +- backend/server/api/parses/ingest.py | 124 +++++++++++---------- backend/server/api/parses/list.py | 83 +++++++------- backend/server/api/parses/list.sql | 33 +++--- backend/server/api/parses/tamper_report.py | 4 +- backend/server/api/rankings.py | 5 +- backend/server/api/rankings.sql | 6 +- backend/server/api/zones.py | 26 +++-- backend/server/api/zones.sql | 7 +- backend/server/db/erasure.py | 60 ++++------ backend/server/metrics.py | 51 ++++++--- backend/server/metrics.sql | 4 +- backend/server/parses/cleanup.py | 98 +++++++++++++--- scripts/dev/seed_fake_parses.py | 18 +-- scripts/dev/seed_raid_kills.py | 29 +++-- tests/conftest.py | 12 +- tests/fixtures/parses_db.py | 64 +++++------ 21 files changed, 372 insertions(+), 312 deletions(-) diff --git a/backend/bot/cogs/parse_posts.py b/backend/bot/cogs/parse_posts.py index 3456ccde..d01b107b 100644 --- a/backend/bot/cogs/parse_posts.py +++ b/backend/bot/cogs/parse_posts.py @@ -22,7 +22,6 @@ import asyncio import logging import os -import sqlite3 import time from datetime import UTC, datetime from typing import TYPE_CHECKING @@ -85,8 +84,6 @@ def collect_new_fights( upload of a fight that STARTED near a candidate, regardless of when it was uploaded — so a mirror that arrives hours or days after the fight was first posted regroups with it and is recognised as already posted.""" - if not parses_db.path.exists(): - return [] # API-layer grouping helpers, imported locally like cleanup.py does — # a module-level import would invert the api→db layering. from backend.server.api.parses.list import ( # noqa: PLC0415 @@ -95,13 +92,14 @@ def collect_new_fights( _group_into_fights, ) + # lower(e.guild_name) = lower(%s) rides idx_encounters_world_guild_lower — + # the Postgres replacement for SQLite's `COLLATE NOCASE` comparison. base = ( f"SELECT e.*, ({_PLAYER_COUNT_SQL}) AS player_count FROM encounters e " - "WHERE e.world = ? AND e.guild_name = ? COLLATE NOCASE AND e.hidden_at IS NULL " + "WHERE e.world = %s AND lower(e.guild_name) = lower(%s) AND e.hidden_at IS NULL " ) conn = parses_db.init_db() try: - conn.row_factory = sqlite3.Row def _kills(rows) -> list[dict]: # Kills only (success_level 1, the rankings rule) — a progression @@ -110,7 +108,7 @@ def _kills(rows) -> list[dict]: candidates = _kills( conn.execute( - base + "AND e.ingested_at > ? AND e.ingested_at <= ? AND e.started_at >= ?", + base + "AND e.ingested_at > %s AND e.ingested_at <= %s AND e.started_at >= %s", (world, guild_name, posted_until, until, until - MAX_FIGHT_AGE_S), ).fetchall() ) @@ -120,16 +118,16 @@ def _kills(rows) -> list[dict]: hi = max(c["started_at"] for c in candidates) + REGROUP_MARGIN_S rows = _kills( conn.execute( - base + "AND e.started_at BETWEEN ? AND ? AND e.ingested_at <= ?", + base + "AND e.started_at BETWEEN %s AND %s AND e.ingested_at <= %s", (world, guild_name, lo, hi, until), ).fetchall() ) for r in rows: if _ensure_classified(conn, r["id"], r.get("zone")): refreshed = conn.execute( - "SELECT COUNT(*) FROM combatants WHERE encounter_id = ? AND is_player = 1", (r["id"],) + "SELECT COUNT(*) AS n FROM combatants WHERE encounter_id = %s AND is_player = 1", (r["id"],) ).fetchone() - r["player_count"] = int(refreshed[0]) + r["player_count"] = int(refreshed["n"]) if refreshed else 0 out: list[tuple[dict, list[dict]]] = [] for g in _group_into_fights(rows, conn): diff --git a/backend/db_catalogue.py b/backend/db_catalogue.py index af6cdd84..3c1c2243 100644 --- a/backend/db_catalogue.py +++ b/backend/db_catalogue.py @@ -402,6 +402,14 @@ def rollback(self) -> None: def close(self) -> None: conn, self._conn = self._conn, None if conn is not None: + # End any implicit read transaction before returning to the pool + # — otherwise psycopg_pool logs a rollback WARNING per checkout. + # Writers commit explicitly; close-without-commit rolls back, + # exactly like sqlite3.Connection.close() did. + try: + conn.rollback() + except Exception: # pragma: no cover — broken conn; pool discards + pass self._pg.putconn(conn) def __enter__(self) -> PgConnProxy: diff --git a/backend/server/api/admin.py b/backend/server/api/admin.py index 03c66911..4a977b0c 100644 --- a/backend/server/api/admin.py +++ b/backend/server/api/admin.py @@ -462,8 +462,6 @@ async def list_parses_admin( world = current_world() def _query() -> list[dict]: - if not parses_db.path.exists(): - return [] conn = parses_db.init_db() try: return parses_db.list_encounters_for_admin( @@ -552,8 +550,6 @@ async def list_tamper_reports_admin( world = current_world() def _query() -> tuple[list[dict], int]: - if not parses_db.path.exists(): - return [], 0 conn = parses_db.init_db() try: rows = parses_db.list_tamper_reports( @@ -617,8 +613,6 @@ async def acknowledge_tamper_report( now_unix = int(datetime.now().timestamp()) def _ack() -> bool: - if not parses_db.path.exists(): - return False conn = parses_db.init_db() try: return parses_db.acknowledge_tamper_report( @@ -661,8 +655,6 @@ async def acknowledge_tamper_reports_batch( now_unix = int(datetime.now().timestamp()) def _ack_batch() -> int: - if not parses_db.path.exists(): - return 0 conn = parses_db.init_db() try: return parses_db.acknowledge_tamper_reports( @@ -701,8 +693,6 @@ async def acknowledge_all_tamper_reports(request: Request) -> AcknowledgeBatchRe now_unix = int(datetime.now().timestamp()) def _ack_all() -> int: - if not parses_db.path.exists(): - return 0 conn = parses_db.init_db() try: return parses_db.acknowledge_all_pending_tamper_reports( @@ -739,8 +729,6 @@ async def purge_acknowledged_tamper_reports(request: Request) -> PurgeResponse: world = current_world() def _purge() -> int: - if not parses_db.path.exists(): - return 0 conn = parses_db.init_db() try: return parses_db.delete_acknowledged_tamper_reports(conn, world) diff --git a/backend/server/api/attendance.py b/backend/server/api/attendance.py index e7b59628..2a34e4e9 100644 --- a/backend/server/api/attendance.py +++ b/backend/server/api/attendance.py @@ -17,7 +17,6 @@ import datetime as dt import logging -import sqlite3 from typing import cast from fastapi import APIRouter, HTTPException, Request @@ -219,8 +218,6 @@ def _parse_roster_sync(world: str, guild_name: str, day: str) -> tuple[str | Non the session merge gap and only the largest cluster counts (the raid night, not an afternoon group). Returns (canonical_guild, members, zones, fight_count) — SYNC, runs in the executor.""" - if not parses_db.path.exists(): - return None, [], [], 0 # API-layer helpers imported locally to keep the module-load DAG # cycle-free (same pattern as cleanup.py / parse_posts.py). from backend.server.api.parses.list import _PLAYER_COUNT_SQL, _ensure_classified # noqa: PLC0415 @@ -229,21 +226,22 @@ def _parse_roster_sync(world: str, guild_name: str, day: str) -> tuple[str | Non # session_day = date(started_at - 6h UTC), so day D covers fights whose # started_at falls in [D 06:00 UTC, D+1 06:00 UTC). win_start = int(dt.datetime(d.year, d.month, d.day, tzinfo=dt.UTC).timestamp()) + ROLLOVER_S + # lower(e.guild_name) = lower(%s) rides idx_encounters_world_guild_lower — + # the Postgres replacement for SQLite's `COLLATE NOCASE` comparison. sql = ( f"SELECT e.id, e.guild_name, e.zone, e.started_at, e.ended_at, ({_PLAYER_COUNT_SQL}) AS player_count " - "FROM encounters e WHERE e.world = ? AND e.guild_name = ? COLLATE NOCASE AND e.hidden_at IS NULL " - "AND e.started_at >= ? AND e.started_at < ?" + "FROM encounters e WHERE e.world = %s AND lower(e.guild_name) = lower(%s) AND e.hidden_at IS NULL " + "AND e.started_at >= %s AND e.started_at < %s" ) conn = parses_db.init_db() try: - conn.row_factory = sqlite3.Row rows = [dict(r) for r in conn.execute(sql, (world, guild_name, win_start, win_start + 86_400)).fetchall()] for r in rows: if _ensure_classified(conn, r["id"], r.get("zone")): refreshed = conn.execute( - "SELECT COUNT(*) FROM combatants WHERE encounter_id = ? AND is_player = 1", (r["id"],) + "SELECT COUNT(*) AS n FROM combatants WHERE encounter_id = %s AND is_player = 1", (r["id"],) ).fetchone() - r["player_count"] = int(refreshed[0]) + r["player_count"] = int(refreshed["n"]) if refreshed else 0 rows = [r for r in rows if (r.get("player_count") or 0) >= _RECONSTRUCT_MIN_PLAYERS] if not rows: return None, [], [], 0 diff --git a/backend/server/api/parses/delete.py b/backend/server/api/parses/delete.py index 2cfc9a00..bcd0dc84 100644 --- a/backend/server/api/parses/delete.py +++ b/backend/server/api/parses/delete.py @@ -15,8 +15,8 @@ from __future__ import annotations import logging -import sqlite3 import time +from typing import Any from fastapi import HTTPException, Request @@ -77,18 +77,16 @@ def _fetch_encounter_auth_rows(ids: list[int], world: str) -> list[dict]: Runs in an executor.""" conn = parses_db.init_db() try: - conn.row_factory = sqlite3.Row - placeholders = ",".join("?" * len(ids)) rows = conn.execute( - f"SELECT id, guild_name, source_dsn, title, hidden_at FROM encounters WHERE id IN ({placeholders}) AND world = ?", - [*ids, world], + "SELECT id, guild_name, source_dsn, title, hidden_at FROM encounters WHERE id = ANY(%s) AND world = %s", + (list(ids), world), ).fetchall() return [dict(r) for r in rows] finally: conn.close() -def _apply_delete(conn: sqlite3.Connection, enc: dict, *, purge: bool, hidden_at: int, hidden_by: str) -> bool: +def _apply_delete(conn: Any, enc: dict, *, purge: bool, hidden_at: int, hidden_by: str) -> bool: """Hard-purge wins; otherwise boss kills are soft-deleted (preserve any ranking) and trash is hard-deleted. Caller has already authorised + (for purge) checked admin. ``hidden_by`` stamps who hid the row for the diff --git a/backend/server/api/parses/ingest.py b/backend/server/api/parses/ingest.py index 6f06cd8e..cb6b24b3 100644 --- a/backend/server/api/parses/ingest.py +++ b/backend/server/api/parses/ingest.py @@ -11,9 +11,10 @@ import hashlib import hmac import logging -import sqlite3 import time +from typing import Any +import psycopg from fastapi import BackgroundTasks, Depends, HTTPException, Request from backend.census.store import store as census_store @@ -338,10 +339,10 @@ def _update_snapshots_sync(encounter_id: int, snapshots: dict[str, CombatantSnap # bucket-fill pool for a different unconfirmed contributor). rows = parses_db.get_combatants_for_encounter(conn, encounter_id) enc = conn.execute( - "SELECT zone FROM encounters WHERE id = ?", + "SELECT zone FROM encounters WHERE id = %s", (encounter_id,), ).fetchone() - zone = enc[0] if enc else None + zone = enc["zone"] if enc else None zone_category = _classify_zone(zone) classification = classify_combatants(rows, zone_category) parses_db.update_combatant_is_player(conn, classification) @@ -630,7 +631,7 @@ def collapse(rows, key_of, table): def _check_idempotency_sync( - conn: sqlite3.Connection, + conn: Any, encid: str, world: str, ) -> tuple[str, int | None, int, int, int] | None: @@ -666,7 +667,7 @@ def _check_idempotency_sync( def _insert_encounter_rows_sync( - conn: sqlite3.Connection, + conn: Any, enc: Encounter, *, combatants: list, @@ -690,59 +691,64 @@ def _insert_encounter_rows_sync( column NULL — the resting state for non-tampered uploads. """ ingested_at = int(time.time()) - with conn: - encounter_id = parses_db.insert_encounter( - conn, - enc, - source_dsn=source_dsn, - ingested_at=ingested_at, - uploaded_by=uploaded_by, - guild_name=guild_name, - world=world, - ) - # Persist client_warnings as JSON on the new row. Sanitise - # defensively: drop empty entries, truncate each at 64 chars - # (matches the plugin's own MaxEntryLength), dedupe. List-length - # is already capped at 32 by IngestRequest's Pydantic validator, - # so a hostile payload doesn't reach us. - cleaned: list[str] = [] - seen: set[str] = set() - for raw in client_warnings or []: - if not isinstance(raw, str): - continue - w = raw.strip() - if not w: - continue - if len(w) > 64: - w = w[:64] - if w in seen: - continue - seen.add(w) - cleaned.append(w) - if cleaned: - import json - - parses_db.set_encounter_client_warnings(conn, encounter_id, json.dumps(cleaned)) - name_to_id = parses_db.insert_combatants_bulk(conn, encounter_id, combatants, snapshots) - n_dt = parses_db.insert_damage_types_bulk(conn, name_to_id, damage_types) - n_at = parses_db.insert_attack_types_bulk(conn, name_to_id, attack_types) - parses_db.mark_ingested( - conn, - enc.encid, - encounter_id, - source_dsn=source_dsn, - ingested_at=ingested_at, - world=world, - ) - # Phase 3 (pet detection): classify ally combatants now that the - # cache-warm snapshot fast-path has populated cls for whatever was - # already in character_cache. Any cls that fills in later via the - # background snapshot resolution triggers a re-classify in - # _update_snapshots_sync. - rows = parses_db.get_combatants_for_encounter(conn, encounter_id) - zone_category = _classify_zone(enc.zone) - classification = classify_combatants(rows, zone_category) - parses_db.update_combatant_is_player(conn, classification) + # One transaction: the statements below ride the connection's open + # transaction and the explicit commit at the end closes it (the old + # sqlite ``with conn:`` scope). On an exception mid-way the transaction + # is left uncommitted; the caller's conn.close() returns the connection + # to the pool, which rolls it back. + encounter_id = parses_db.insert_encounter( + conn, + enc, + source_dsn=source_dsn, + ingested_at=ingested_at, + uploaded_by=uploaded_by, + guild_name=guild_name, + world=world, + ) + # Persist client_warnings as JSON on the new row. Sanitise + # defensively: drop empty entries, truncate each at 64 chars + # (matches the plugin's own MaxEntryLength), dedupe. List-length + # is already capped at 32 by IngestRequest's Pydantic validator, + # so a hostile payload doesn't reach us. + cleaned: list[str] = [] + seen: set[str] = set() + for raw in client_warnings or []: + if not isinstance(raw, str): + continue + w = raw.strip() + if not w: + continue + if len(w) > 64: + w = w[:64] + if w in seen: + continue + seen.add(w) + cleaned.append(w) + if cleaned: + import json + + parses_db.set_encounter_client_warnings(conn, encounter_id, json.dumps(cleaned)) + name_to_id = parses_db.insert_combatants_bulk(conn, encounter_id, combatants, snapshots) + n_dt = parses_db.insert_damage_types_bulk(conn, name_to_id, damage_types) + n_at = parses_db.insert_attack_types_bulk(conn, name_to_id, attack_types) + parses_db.mark_ingested( + conn, + enc.encid, + encounter_id, + source_dsn=source_dsn, + ingested_at=ingested_at, + world=world, + ) + # Phase 3 (pet detection): classify ally combatants now that the + # cache-warm snapshot fast-path has populated cls for whatever was + # already in character_cache. Any cls that fills in later via the + # background snapshot resolution triggers a re-classify in + # _update_snapshots_sync. + rows = parses_db.get_combatants_for_encounter(conn, encounter_id) + zone_category = _classify_zone(enc.zone) + classification = classify_combatants(rows, zone_category) + parses_db.update_combatant_is_player(conn, classification) + conn.commit() return encounter_id, n_dt, n_at @@ -1156,7 +1162,7 @@ async def ingest_parse( snapshots, parse_world, ) - except (DuplicatePayloadRows, sqlite3.IntegrityError) as exc: + except (DuplicatePayloadRows, psycopg.errors.UniqueViolation) as exc: # A payload that breaks a uniqueness rule is the CLIENT's bug: say # so with a 422 (a 500 reads as "server broke, retry" and one client # did exactly that, every ~2 s, for nine hours on 2026-09-27). Log diff --git a/backend/server/api/parses/list.py b/backend/server/api/parses/list.py index 92979438..b783b50d 100644 --- a/backend/server/api/parses/list.py +++ b/backend/server/api/parses/list.py @@ -10,11 +10,11 @@ import asyncio import functools import logging -import sqlite3 from collections.abc import Callable, Mapping from types import MappingProxyType -from typing import Literal +from typing import Any, Literal +import psycopg from fastapi import HTTPException, Request from backend.eq2db.zones import catalogue as zones_db @@ -93,7 +93,7 @@ def _uploader_discord_id(source_dsn: str | None) -> str | None: _ALL_ALLY_SQL = _SQL["all_ally_names"] -def _ensure_classified(conn: sqlite3.Connection, encounter_id: int, zone: str | None) -> bool: +def _ensure_classified(conn: Any, encounter_id: int, zone: str | None) -> bool: """Lazy backfill for pre-Phase-4 combatant rows. If any combatant for this encounter has ``is_player IS NULL`` (i.e. @@ -115,24 +115,22 @@ def _ensure_classified(conn: sqlite3.Connection, encounter_id: int, zone: str | return _classify_now(conn, encounter_id, zone) -def encounters_needing_classification(conn: sqlite3.Connection, encounter_ids: list[int]) -> set[int]: +def encounters_needing_classification(conn: Any, encounter_ids: list[int]) -> set[int]: """Batched probe: which of these encounters still have unclassified - ally rows? One chunked query instead of one probe per encounter — at - Wuoshi's backlog size the per-encounter probes alone cost the rankings - rebuild the better part of a minute.""" - out: set[int] = set() - chunk_size = 500 - for i in range(0, len(encounter_ids), chunk_size): - chunk = encounter_ids[i : i + chunk_size] - rows = conn.execute( - _SQL["encounters_with_unclassified_combatants"].format(placeholders=",".join("?" * len(chunk))), - chunk, - ).fetchall() - out.update(r[0] for r in rows) - return out - - -def _classify_now(conn: sqlite3.Connection, encounter_id: int, zone: str | None) -> bool: + ally rows? ONE ``= ANY`` query (the SQLite 500-id chunking is gone) + instead of one probe per encounter — at Wuoshi's backlog size the + per-encounter probes alone cost the rankings rebuild the better part + of a minute.""" + if not encounter_ids: + return set() + rows = conn.execute( + _SQL["encounters_with_unclassified_combatants"], + (list(encounter_ids),), + ).fetchall() + return {r["encounter_id"] for r in rows} + + +def _classify_now(conn: Any, encounter_id: int, zone: str | None) -> bool: """The classify-and-persist half of _ensure_classified, probe already done — callers using the BATCHED probe skip the per-encounter one.""" rows = parses_db.get_combatants_for_encounter(conn, encounter_id) @@ -141,17 +139,18 @@ def _classify_now(conn: sqlite3.Connection, encounter_id: int, zone: str | None) try: parses_db.update_combatant_is_player(conn, classification) conn.commit() - except sqlite3.OperationalError as exc: - # Backfill is best-effort maintenance riding a READ path — a lock - # collision with concurrent ingest must degrade to "try again next - # read", never take the rankings/parses response down with it. + except psycopg.Error as exc: + # Backfill is best-effort maintenance riding a READ path — a write + # collision with concurrent ingest (deadlock/serialization failure) + # must degrade to "try again next read", never take the + # rankings/parses response down with it. _log.warning("[parses-list] lazy classify skipped for encounter %s: %s", encounter_id, exc) conn.rollback() return False return True -def _top_n_ally_names(conn: sqlite3.Connection, encounter_id: int, n: int) -> set[str]: +def _top_n_ally_names(conn: Any, encounter_id: int, n: int) -> set[str]: """Return the top-N player names in this encounter by encDPS descending. Tiebreaker on name ASC so two combatants with identical encDPS pick the @@ -162,14 +161,14 @@ def _top_n_ally_names(conn: sqlite3.Connection, encounter_id: int, n: int) -> se allies than ``n``. Empty set when there are no qualifying allies at all (e.g. an empty-ally parse) — that case still merges trivially under the Phase 4 mutual-containment rule (``set() ⊆ X`` is always true).""" - return {row[0] for row in conn.execute(_TOP_N_ALLY_SQL, (encounter_id, n))} + return {row["name"] for row in conn.execute(_TOP_N_ALLY_SQL, (encounter_id, n))} -def _all_ally_names(conn: sqlite3.Connection, encounter_id: int) -> set[str]: +def _all_ally_names(conn: Any, encounter_id: int) -> set[str]: """Every qualifying player name in the encounter. Pairs with ``_top_n_ally_names`` to evaluate the merger's mutual-containment rule (``top_N(A) ⊆ allies(B)`` and vice versa).""" - return {row[0] for row in conn.execute(_ALL_ALLY_SQL, (encounter_id,))} + return {row["name"] for row in conn.execute(_ALL_ALLY_SQL, (encounter_id,))} # ── Zone classifier ────────────────────────────────────────────────────── @@ -269,22 +268,20 @@ def _list_encounters_sync( ``world`` scopes results to the active server so a Varsoon viewer only sees Varsoon parses. ``before`` is the pagination cursor: only rows strictly older than that unix timestamp.""" - if not parses_db.path.exists(): - return [] - # Soft-deleted parses are hidden from the list (but still feed rankings). - # Note: the WHERE clause operates on the outer query's columns (no alias). - where_clauses: list[str] = ["hidden_at IS NULL", "world = ?"] + # Note: the WHERE clause operates on the derived table's columns (no + # `e.` prefix — the subquery is aliased `sub` in the .sql block). + where_clauses: list[str] = ["hidden_at IS NULL", "world = %s"] params: list = [world] if before is not None: - where_clauses.append("started_at < ?") + where_clauses.append("started_at < %s") params.append(before) if zone: - where_clauses.append("zone = ?") + where_clauses.append("zone = %s") params.append(zone) if size and size in SIZE_BUCKETS: lo, hi = SIZE_BUCKETS[size] - where_clauses.append("player_count BETWEEN ? AND ?") + where_clauses.append("player_count BETWEEN %s AND %s") params.extend([lo, hi]) if search and search.strip(): # Free-text filter over the user-visible fields, mirroring the admin @@ -292,8 +289,8 @@ def _list_encounters_sync( # so a whole fight matches when any of its uploads does. like = f"%{search.strip().lower()}%" where_clauses.append( - "(LOWER(title) LIKE ? OR LOWER(IFNULL(zone, '')) LIKE ? " - "OR LOWER(IFNULL(uploaded_by, '')) LIKE ? OR LOWER(IFNULL(guild_name, '')) LIKE ?)" + "(LOWER(title) LIKE %s OR LOWER(COALESCE(zone, '')) LIKE %s " + "OR LOWER(COALESCE(uploaded_by, '')) LIKE %s OR LOWER(COALESCE(guild_name, '')) LIKE %s)" ) params.extend([like, like, like, like]) where_sql = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else "" @@ -302,13 +299,12 @@ def _list_encounters_sync( conn = parses_db.init_db() try: - conn.row_factory = sqlite3.Row return [dict(r) for r in conn.execute(list_sql, [*params, inner_cap]).fetchall()] finally: conn.close() -def _group_into_fights(encounters: list[dict], conn: sqlite3.Connection) -> list[dict]: +def _group_into_fights(encounters: list[dict], conn: Any) -> list[dict]: """Greedy mirror-grouping. Two uploads are the same fight when ALL of: - they come from *different* uploaders, - their guild + title match, @@ -422,11 +418,8 @@ def _encounter_detail_sync(encounter_id: int, top_attacks_per_combatant: int, wo ``world`` is used to scope the lookup so a viewer on one server can't read another server's encounter by guessing its integer id.""" - if not parses_db.path.exists(): - return None conn = parses_db.init_db() try: - conn.row_factory = sqlite3.Row enc_row = conn.execute(_SQL["select_encounter_by_id_and_world"], (encounter_id, world)).fetchone() if enc_row is None: return None @@ -561,10 +554,10 @@ def _build_list_dataset( for r in rows: if _ensure_classified(conn, r["id"], r.get("zone")): refreshed = conn.execute( - "SELECT COUNT(*) FROM combatants WHERE encounter_id = ? AND is_player = 1", + "SELECT COUNT(*) AS n FROM combatants WHERE encounter_id = %s AND is_player = 1", (r["id"],), ).fetchone() - r["player_count"] = int(refreshed[0]) + r["player_count"] = int(refreshed["n"]) fights = _group_into_fights(rows, conn) finally: conn.close() diff --git a/backend/server/api/parses/list.sql b/backend/server/api/parses/list.sql index 8f51a52f..54b2ec0c 100644 --- a/backend/server/api/parses/list.sql +++ b/backend/server/api/parses/list.sql @@ -1,4 +1,5 @@ --- SQL for backend/server/api/parses/list.py — encounter listing + detail. +-- SQL for backend/server/api/parses/list.py — encounter listing + detail +-- (psycopg, parses schema). -- -- player_count_subquery is exposed as a Python name (_PLAYER_COUNT_SQL) for -- rankings.py which composes it into its own .sql via .format() — keep it @@ -29,13 +30,13 @@ WHERE c.encounter_id = e.id AND c.is_player = 1 -- deterministic tiebreaks. Used by the merger to evaluate mutual -- containment of two uploads' top-N sets. SELECT name FROM combatants -WHERE encounter_id = ? AND is_player = 1 +WHERE encounter_id = %s AND is_player = 1 ORDER BY encdps DESC, name ASC -LIMIT ?; +LIMIT %s; -- :name all_ally_names SELECT name FROM combatants -WHERE encounter_id = ? AND is_player = 1; +WHERE encounter_id = %s AND is_player = 1; -- --------------------------------------------------------------------------- -- Lazy combatant-classification trigger @@ -52,36 +53,38 @@ WHERE encounter_id = ? AND is_player = 1; -- load (thousands of write+commit cycles per cold cache fill, colliding -- with raid-night ingest → "database is locked"). SELECT 1 FROM combatants -WHERE encounter_id = ? AND ally = 1 AND is_player IS NULL LIMIT 1; +WHERE encounter_id = %s AND ally = 1 AND is_player IS NULL LIMIT 1; -- :name encounters_with_unclassified_combatants --- {placeholders} = comma-joined "?,?,..." for the IN list. Batched form of --- has_unclassified_combatants: the rankings rebuild probes its whole --- candidate set in a handful of queries instead of one per encounter. +-- Batched form of has_unclassified_combatants: the rankings rebuild probes +-- its whole candidate set in ONE = ANY query (the SQLite 500-id IN-list +-- chunking is gone) instead of one probe per encounter. SELECT DISTINCT encounter_id FROM combatants -WHERE encounter_id IN ({placeholders}) AND ally = 1 AND is_player IS NULL; +WHERE encounter_id = ANY(%s) AND ally = 1 AND is_player IS NULL; -- --------------------------------------------------------------------------- -- Encounter list + detail -- --------------------------------------------------------------------------- --- :name list_encounters_recent --- Encounter rows most-recent-first, capped at the inner ? limit. Caller +-- Encounter rows most-recent-first, capped at the inner %s limit. Caller -- composes the WHERE filter from request params and templates both -- where_sql (the full "WHERE ... AND ...") and player_count_sql (the --- correlated subquery) in via .format(). +-- correlated subquery) in via .format(). (Comment kept OUTSIDE the block — +-- the composed where_sql contains %s and psycopg counts placeholders +-- inside comments.) +-- :name list_encounters_recent SELECT * FROM ( SELECT e.*, ({player_count_sql}) AS player_count, (SELECT COUNT(*) FROM combatants c2 WHERE c2.encounter_id = e.id) AS combatant_count FROM encounters e -) +) sub {where_sql} ORDER BY started_at DESC -LIMIT ?; +LIMIT %s; -- :name select_encounter_by_id_and_world -- World-scoped fetch by encounter id. The world scope keeps a viewer on -- one server from reading another server's encounter by guessing its -- integer id. -SELECT * FROM encounters WHERE id = ? AND world = ?; +SELECT * FROM encounters WHERE id = %s AND world = %s; diff --git a/backend/server/api/parses/tamper_report.py b/backend/server/api/parses/tamper_report.py index 86d8043c..087f7ce5 100644 --- a/backend/server/api/parses/tamper_report.py +++ b/backend/server/api/parses/tamper_report.py @@ -27,9 +27,9 @@ from __future__ import annotations import logging -import sqlite3 import time +import psycopg from fastapi import HTTPException, Request from backend.core.log_safety import scrub as _safe_for_log @@ -245,7 +245,7 @@ async def report_tamper( uploader_discord_name=discord_name, payload_json=payload_json, ) - except sqlite3.Error: + except psycopg.Error: # Drop discord_id from the diagnostic log — encid + reason are # the actionable bits, and the actor's identity is in the DB # row when the insert succeeded (and on every other call in diff --git a/backend/server/api/rankings.py b/backend/server/api/rankings.py index 8c22a646..1a4fc12c 100644 --- a/backend/server/api/rankings.py +++ b/backend/server/api/rankings.py @@ -656,14 +656,11 @@ def _load_primary_boss_kills(world: str = "Varsoon") -> list[dict]: ``world`` scopes to the active server so each server sees only its own leaderboard data.""" - if not parses_db.path.exists(): - return [] import time as _time # noqa: PLC0415 — phase timings for the rebuild log t0 = _time.monotonic() conn = parses_db.init_db() try: - conn.row_factory = sqlite3.Row rows = conn.execute( _SQL["list_winning_encounters_with_player_count"].format(player_count_sql=_PLAYER_COUNT_SQL), (world,), @@ -688,7 +685,7 @@ def _load_primary_boss_kills(world: str = "Varsoon") -> list[dict]: _SQL["count_player_combatants_for_encounter"], (r["id"],), ).fetchone() - r["player_count"] = int(refreshed[0]) + r["player_count"] = int(refreshed["n"]) if refreshed else 0 t_classify = _time.monotonic() # Gate + canonicalise per row (scope is known from player_count): raid # bosses resolve against zones.db, everything else via the heuristic. diff --git a/backend/server/api/rankings.sql b/backend/server/api/rankings.sql index 13092e8d..d78dd503 100644 --- a/backend/server/api/rankings.sql +++ b/backend/server/api/rankings.sql @@ -44,7 +44,7 @@ SELECT encounter_name FROM zone_encounters WHERE zone_id = ? ORDER BY position; -- --------------------------------------------------------------------------- --- parses.db — winning-encounters scan + player_count refresh +-- parses schema (Postgres) — winning-encounters scan + player_count refresh -- --------------------------------------------------------------------------- -- :name list_winning_encounters_with_player_count @@ -55,10 +55,10 @@ SELECT e.id, e.title, e.zone, e.guild_name, e.uploaded_by, e.started_at, e.duration_s, e.success_level, e.ingested_at, ({player_count_sql}) AS player_count FROM encounters e -WHERE e.success_level = 1 AND e.world = ? +WHERE e.success_level = 1 AND e.world = %s ORDER BY e.started_at DESC; -- :name count_player_combatants_for_encounter -- Refresh the player count for one encounter after the lazy backfill -- classifies its combatants. -SELECT COUNT(*) FROM combatants WHERE encounter_id = ? AND is_player = 1; +SELECT COUNT(*) AS n FROM combatants WHERE encounter_id = %s AND is_player = 1; diff --git a/backend/server/api/zones.py b/backend/server/api/zones.py index 9360ff1f..eb22a529 100644 --- a/backend/server/api/zones.py +++ b/backend/server/api/zones.py @@ -267,12 +267,12 @@ async def _resolve_primary_guild(discord_id: str) -> tuple[str | None, str | Non def _most_recent_parsed_guild_sync(discord_id: str) -> str | None: """Most recent non-null guild_name this user has uploaded a parse for.""" - if not parses_db.path.exists(): - return None - with sqlite3.connect(parses_db.path) as conn: - conn.execute("PRAGMA query_only = ON") + conn = parses_db.init_db() + try: row = conn.execute(_SQL["most_recent_parsed_guild"], (discord_id,)).fetchone() - return row[0] if row else None + finally: + conn.close() + return row["guild_name"] if row else None def _compute_progress_sync(guild_name: str) -> dict[str, list[KilledEncounter]]: @@ -285,22 +285,24 @@ def _compute_progress_sync(guild_name: str) -> dict[str, list[KilledEncounter]]: ``zone_encounter_mobs.mob_name_lower`` — solo bosses match directly, group encounters collapse (any one mob ACT logs counts the whole encounter as cleared). Aggregation happens in Python rather than SQL because the parses - and zones data live in separate SQLite files and we'd rather avoid - cross-DB ATTACH. + data lives in Postgres while the zones data is a local SQLite file — a + Python stitch beats any cross-database join contraption. """ - if not parses_db.path.exists() or not zones_db.path.exists(): + if not zones_db.path.exists(): return {} # Pull every winning row for the guild as (id, title_lower, started_at). # We need the timestamp + id to surface "last kill" — a DISTINCT title pass # wouldn't be enough. - with sqlite3.connect(parses_db.path) as pconn: - pconn.execute("PRAGMA query_only = ON") + pconn = parses_db.init_db() + try: kills = [ - (row[0], row[1].lower(), row[2]) + (row["id"], row["title"].lower(), row["started_at"]) for row in pconn.execute(_SQL["list_kills_for_guild"], (guild_name,)).fetchall() - if row[1] + if row["title"] ] + finally: + pconn.close() if not kills: return {} diff --git a/backend/server/api/zones.sql b/backend/server/api/zones.sql index 3ca15fae..2d203b2b 100644 --- a/backend/server/api/zones.sql +++ b/backend/server/api/zones.sql @@ -8,15 +8,16 @@ -- :name most_recent_parsed_guild -- Most recent non-null guild_name this user has uploaded a parse for. +-- Runs against the Postgres parses schema (%s placeholders). SELECT guild_name FROM encounters -WHERE uploaded_by = ? AND guild_name IS NOT NULL AND hidden_at IS NULL +WHERE uploaded_by = %s AND guild_name IS NOT NULL AND hidden_at IS NULL ORDER BY started_at DESC LIMIT 1; -- :name list_kills_for_guild -- Every winning row for a guild as (id, title, started_at). Caller filters --- out NULL titles in Python. +-- out NULL titles in Python. Postgres parses schema (%s placeholders). SELECT id, title, started_at FROM encounters -WHERE guild_name = ? AND success_level = 1 AND hidden_at IS NULL; +WHERE guild_name = %s AND success_level = 1 AND hidden_at IS NULL; -- :name match_encounter_mobs_by_titles_chunk -- For a chunk of mob_lower titles, resolve each to its (zone, encounter) diff --git a/backend/server/db/erasure.py b/backend/server/db/erasure.py index 30447e14..277c9451 100644 --- a/backend/server/db/erasure.py +++ b/backend/server/db/erasure.py @@ -1,9 +1,9 @@ """Account erasure — the right-to-be-forgotten mechanics behind the privacy policy (2026-09-28). -One sync function over BOTH stores — the ``users`` Postgres schema and the -parses SQLite file (Postgres in P2, when the halves unify into one -transaction) — run from the routes through ``run_sync``: +One sync function, ONE Postgres transaction spanning the ``users`` and +``parses`` schemas (search_path switched mid-transaction) — run from the +routes through ``run_sync``: - rows that ARE the person go: the ``users`` row and everything keyed to their Discord id (tokens, roles, role requests, claims, favourites, @@ -27,16 +27,14 @@ from __future__ import annotations -import sqlite3 import time from dataclasses import dataclass, field -from pathlib import Path from psycopg.types.json import Json from backend import pg from backend.server import db as _users_db # SCHEMA read at call time (tests re-point it) -from backend.server.parses.db import store as parses_store +from backend.server.parses.db import SCHEMA as _PARSES_SCHEMA #: The placeholder users row every tombstoned reference points at. DELETED_USER_ID = "deleted" @@ -100,10 +98,6 @@ def as_dict(self) -> dict: } -def _table_exists(conn: sqlite3.Connection, table: str) -> bool: - return conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone() is not None - - def _ensure_tombstone(conn, now: int) -> None: conn.execute( "INSERT INTO users (discord_id, discord_name, discord_username, avatar, first_seen, last_seen, " @@ -117,15 +111,15 @@ def erase_user_sync( discord_id: str, *, users_schema: str | None = None, - parses_path: Path | None = None, + parses_schema: str | None = None, now: int | None = None, ) -> ErasureResult: - """Erase one Discord account from both stores. Idempotent: a second + """Erase one Discord account from both schemas. Idempotent: a second call finds nothing and reports ``found=False``. The tombstone id itself - can never be erased. The users half is ONE Postgres transaction with - constraints deferred (replacing SQLite's PRAGMA foreign_keys=OFF), so a - crash mid-way leaves nothing half-erased; migrations guarantee every - table exists, so the SQLite-era per-table existence probes are gone.""" + can never be erased. The WHOLE erasure is one Postgres transaction with + users constraints deferred (replacing SQLite's PRAGMA foreign_keys=OFF), + so a crash mid-way leaves nothing half-erased; migrations guarantee + every table exists, so the per-table existence probes are gone.""" if not discord_id or discord_id == DELETED_USER_ID: return ErasureResult(found=False) now = int(now if now is not None else time.time()) @@ -165,26 +159,20 @@ def erase_user_sync( cur = conn.execute("DELETE FROM users WHERE discord_id = %s", (discord_id,)) if cur.rowcount: result.deleted["users"] = cur.rowcount - conn.commit() - parses_db = Path(parses_path) if parses_path is not None else parses_store.path - if parses_db.exists(): - conn = parses_store.init_db() if parses_path is None else sqlite3.connect(parses_db) - try: - dsn = f"plugin:{discord_id}" - cur = conn.execute("UPDATE encounters SET source_dsn = ? WHERE source_dsn = ?", (DELETED_SOURCE_DSN, dsn)) - result.parses_anonymised = cur.rowcount - if _table_exists(conn, "ingest_log"): - conn.execute("UPDATE ingest_log SET source_dsn = ? WHERE source_dsn = ?", (DELETED_SOURCE_DSN, dsn)) - conn.execute("UPDATE encounters SET hidden_by = NULL WHERE hidden_by = ?", (discord_id,)) - if _table_exists(conn, "tamper_reports"): - cur = conn.execute("DELETE FROM tamper_reports WHERE uploader_discord_id = ?", (discord_id,)) - result.tamper_reports_deleted = cur.rowcount - conn.execute( - "UPDATE tamper_reports SET acknowledged_by = NULL WHERE acknowledged_by = ?", (discord_id,) - ) - conn.commit() - finally: - conn.close() + # ---- parses half: SAME transaction, schema switched in place. + # SET search_path is transactional, so the whole erasure commits or + # rolls back as one unit — the old users.db/parses.db split could + # crash between the halves and leave a half-erased account. + conn.execute(pg.search_path_sql(parses_schema if parses_schema is not None else _PARSES_SCHEMA)) + dsn = f"plugin:{discord_id}" + cur = conn.execute("UPDATE encounters SET source_dsn = %s WHERE source_dsn = %s", (DELETED_SOURCE_DSN, dsn)) + result.parses_anonymised = cur.rowcount + conn.execute("UPDATE ingest_log SET source_dsn = %s WHERE source_dsn = %s", (DELETED_SOURCE_DSN, dsn)) + conn.execute("UPDATE encounters SET hidden_by = NULL WHERE hidden_by = %s", (discord_id,)) + cur = conn.execute("DELETE FROM tamper_reports WHERE uploader_discord_id = %s", (discord_id,)) + result.tamper_reports_deleted = cur.rowcount + conn.execute("UPDATE tamper_reports SET acknowledged_by = NULL WHERE acknowledged_by = %s", (discord_id,)) + conn.commit() return result diff --git a/backend/server/metrics.py b/backend/server/metrics.py index 1f48fd95..c3a1dfe9 100644 --- a/backend/server/metrics.py +++ b/backend/server/metrics.py @@ -290,19 +290,18 @@ def collect(self): # type: ignore[override] except Exception: _log.exception("[metrics] users-schema collector error") - # parses.db — encounters split by hidden_at (visible vs soft-deleted) - # so dashboards can distinguish "live leaderboard rows" from - # accumulated history. - conn = self._get_conn("parses", parses_db.DB_PATH) - if conn is not None: - try: + # parses schema (Postgres) — encounters split by hidden_at (visible vs + # soft-deleted) so dashboards can distinguish "live leaderboard rows" + # from accumulated history. + try: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(parses_db.SCHEMA)) row = conn.execute(_SQL["count_visible_encounters"]).fetchone() - g_parses.add_metric(["visible"], row[0] if row else 0) + g_parses.add_metric(["visible"], row["n"] if row else 0) row = conn.execute(_SQL["count_hidden_encounters"]).fetchone() - g_parses.add_metric(["hidden"], row[0] if row else 0) - except Exception: - _log.exception("[metrics] parses.db collector error") - self._close_conn("parses") + g_parses.add_metric(["hidden"], row["n"] if row else 0) + except Exception: + _log.exception("[metrics] parses-schema collector error") # raids.db — strategies + the ACT trigger pack. conn = self._get_conn("raids", raids_db.DB_PATH) @@ -344,13 +343,11 @@ def collect(self): # type: ignore[override] from backend.eq2db import recipes as recipes_db from backend.eq2db import spells as spells_db from backend.eq2db import zones as zones_db - from backend.server.parses import db as parses_db # Map label → Path. Centralised so adding a new DB is one tuple. - # (users moved to Postgres — its size gauge returns in the P2 - # metrics split as pg_total_relation_size per schema.) + # (users + parses moved to Postgres — covered by the schema-size + # gauge below instead of a file stat.) candidates = [ - ("parses", parses_db.DB_PATH), ("census", census_store.DB_PATH), ("raids", raids_db.DB_PATH), ("zones", zones_db.DB_PATH), @@ -375,6 +372,30 @@ def collect(self): # type: ignore[override] yield g_size + # Migrated families: total relation size per Postgres schema — the + # growth-trend replacement for their old file gauges (and the + # measurement behind the Supabase tier decision). + g_pg = GaugeMetricFamily( + "pg_schema_size_bytes", + "Total relation size per migrated Postgres family schema (bytes)", + labels=["schema"], + ) + try: + from backend import pg + + with pg.connection() as conn: + rows = conn.execute( + "SELECT schemaname AS s," + " SUM(pg_total_relation_size((quote_ident(schemaname) || '.' || quote_ident(tablename))::regclass))::bigint AS b" + " FROM pg_tables WHERE schemaname IN ('users', 'parses', 'census', 'zones', 'raids')" + " GROUP BY schemaname" + ).fetchall() + for r in rows: + g_pg.add_metric([r["s"]], r["b"] or 0) + except Exception: + _log.exception("[metrics] pg schema-size collector error") + yield g_pg + class _CensusHealthCollector(Collector): """Read the in-memory census-health state at scrape time and surface it diff --git a/backend/server/metrics.sql b/backend/server/metrics.sql index f746e68b..bee27aca 100644 --- a/backend/server/metrics.sql +++ b/backend/server/metrics.sql @@ -7,10 +7,10 @@ SELECT COUNT(*) AS n FROM users WHERE access_status = %s; SELECT COUNT(*) AS n FROM character_claims WHERE status = %s; -- :name count_visible_encounters -SELECT COUNT(*) FROM encounters WHERE hidden_at IS NULL; +SELECT COUNT(*) AS n FROM encounters WHERE hidden_at IS NULL; -- :name count_hidden_encounters -SELECT COUNT(*) FROM encounters WHERE hidden_at IS NOT NULL; +SELECT COUNT(*) AS n FROM encounters WHERE hidden_at IS NOT NULL; -- :name count_raid_encounters SELECT COUNT(*) FROM raid_encounters; diff --git a/backend/server/parses/cleanup.py b/backend/server/parses/cleanup.py index 1d3a11f9..ce445abd 100644 --- a/backend/server/parses/cleanup.py +++ b/backend/server/parses/cleanup.py @@ -1,6 +1,6 @@ -"""Background retention sweep for parses. +"""Background retention sweeps for parses. -Two rules, both keyed on the fight time (``encounters.started_at``): +Row retention (both rules keyed on the fight time, ``encounters.started_at``): * **Trash** encounters (non-boss — title starts lowercase, see ``boss.is_boss``) are hard-deleted ``RETENTION_DAYS`` after the fight. @@ -8,6 +8,21 @@ uploads are cleared ``RETENTION_DAYS`` after the fight — only the primary (the longest-duration canonical, the upload rankings link to) is kept. +Detail retention (user decision 2026-10-04 — the 6.8GB problem): the kept +fights' per-ability breakdown rows (``attack_types`` + ``damage_types``) +are dropped on a tier schedule by zone category (``_classify_zone`` — the +same curated-zones bucketing the parses list uses): + + * curated RAID zones ............ ``DETAIL_RETENTION_RAID_DAYS`` (30) + * curated group-instance zones .. ``DETAIL_RETENTION_DUNGEON_DAYS`` (14) + * anything else ................. ``DETAIL_RETENTION_OTHER_DAYS`` (7) + +``encounters`` + ``combatants`` (everything rankings, the parse list and +character pages read) stay forever; ``detail_pruned_at`` is stamped so the +parse detail page can show a "breakdown pruned" notice instead of silently +empty tables. Trash dies at 3 days, so by the shortest tier only boss +fights remain — no is_boss re-check needed here. + Hard-deleting the non-primary boss uploads is rankings-safe: rankings (``rankings._load_primary_boss_kills``) already mirror-group via the same ``_group_into_fights`` and use only the primary upload, and leaderboard links @@ -25,7 +40,6 @@ import logging import os -import sqlite3 import time from backend.server.parses.boss import is_boss @@ -34,17 +48,25 @@ _log = logging.getLogger(__name__) RETENTION_DAYS = int(os.getenv("PARSE_RETENTION_DAYS", "3")) + +#: Tiered attack_types/damage_types retention by zone category. +DETAIL_RETENTION_RAID_DAYS = int(os.getenv("PARSE_DETAIL_RETENTION_RAID_DAYS", "30")) +DETAIL_RETENTION_DUNGEON_DAYS = int(os.getenv("PARSE_DETAIL_RETENTION_DUNGEON_DAYS", "14")) +DETAIL_RETENTION_OTHER_DAYS = int(os.getenv("PARSE_DETAIL_RETENTION_OTHER_DAYS", "7")) + _DAY_S = 86_400 +#: Prune in chunks so one DELETE never holds a giant transaction. +_PRUNE_CHUNK = 200 -def run_parse_cleanup(now: int | None = None, retention_days: int | None = None) -> dict[str, int]: - """Delete aged trash and collapse aged boss mirror-groups to their primary. - Returns ``{"trash_deleted": n, "dup_uploads_deleted": m}``. Safe to call - repeatedly (idempotent once nothing is past the cutoff).""" - if not parses_db.path.exists(): - return {"trash_deleted": 0, "dup_uploads_deleted": 0} +def run_parse_cleanup(now: int | None = None, retention_days: int | None = None) -> dict[str, int]: + """Delete aged trash, collapse aged boss mirror-groups to their primary, + then drop overdue breakdown detail per the retention tiers. + Returns ``{"trash_deleted": n, "dup_uploads_deleted": m, + "detail_pruned": k}``. Safe to call repeatedly (idempotent once nothing + is past the cutoffs).""" now = int(time.time()) if now is None else now days = RETENTION_DAYS if retention_days is None else retention_days cutoff = now - days * _DAY_S @@ -61,15 +83,14 @@ def run_parse_cleanup(now: int | None = None, retention_days: int | None = None) # Same player_count projection list/rankings use, so grouping (and therefore # the chosen primary) matches exactly. `e` is the encounters alias. candidate_sql = ( - f"SELECT e.*, ({_PLAYER_COUNT_SQL}) AS player_count FROM encounters e WHERE e.world = ? AND e.started_at < ?" + f"SELECT e.*, ({_PLAYER_COUNT_SQL}) AS player_count FROM encounters e WHERE e.world = %s AND e.started_at < %s" ) trash_deleted = 0 dup_deleted = 0 conn = parses_db.init_db() try: - conn.row_factory = sqlite3.Row - worlds = [r[0] for r in conn.execute("SELECT DISTINCT world FROM encounters").fetchall()] + worlds = [r["world"] for r in conn.execute("SELECT DISTINCT world FROM encounters").fetchall()] for world in worlds: rows = [dict(r) for r in conn.execute(candidate_sql, (world, cutoff)).fetchall()] @@ -89,10 +110,10 @@ def run_parse_cleanup(now: int | None = None, retention_days: int | None = None) for r in boss_rows: if _ensure_classified(conn, r["id"], r.get("zone")): refreshed = conn.execute( - "SELECT COUNT(*) FROM combatants WHERE encounter_id = ? AND is_player = 1", + "SELECT COUNT(*) AS n FROM combatants WHERE encounter_id = %s AND is_player = 1", (r["id"],), ).fetchone() - r["player_count"] = int(refreshed[0]) + r["player_count"] = int(refreshed["n"]) if refreshed else 0 for g in _group_into_fights(boss_rows, conn): # Keep the canonical (longest overall) AND the longest winning @@ -106,14 +127,57 @@ def run_parse_cleanup(now: int | None = None, retention_days: int | None = None) continue if parses_db.delete_encounter(conn, u["id"]): dup_deleted += 1 + + detail_pruned = _sweep_detail_retention(conn, now) finally: conn.close() - if trash_deleted or dup_deleted: + if trash_deleted or dup_deleted or detail_pruned: _log.info( - "[parse-cleanup] swept trash_deleted=%s dup_uploads_deleted=%s (retention=%sd)", + "[parse-cleanup] swept trash_deleted=%s dup_uploads_deleted=%s detail_pruned=%s (retention=%sd)", trash_deleted, dup_deleted, + detail_pruned, days, ) - return {"trash_deleted": trash_deleted, "dup_uploads_deleted": dup_deleted} + return {"trash_deleted": trash_deleted, "dup_uploads_deleted": dup_deleted, "detail_pruned": detail_pruned} + + +def detail_retention_days(zone: str | None) -> int: + """The breakdown-retention tier (in days) for a parse's zone — curated + raid zones keep full detail longest, curated group instances less, + everything else least. Import-light wrapper over the parses list's + ``_classify_zone`` so the copy script and tests share one rule.""" + from backend.server.api.parses.list import _classify_zone # noqa: PLC0415 + + category = _classify_zone(zone) + if category == "raid": + return DETAIL_RETENTION_RAID_DAYS + if category == "dungeon": + return DETAIL_RETENTION_DUNGEON_DAYS + return DETAIL_RETENTION_OTHER_DAYS + + +def _sweep_detail_retention(conn, now: int) -> int: + """Drop attack_types/damage_types for every kept encounter whose tier + cutoff has passed; stamp ``detail_pruned_at``. Returns encounters pruned. + + Candidates = detail not yet pruned AND older than the SHORTEST tier — + cheap via the partial idx_encounters_detail_prune index. Each candidate + is then held against its own zone's tier, so a 10-day-old curated raid + kill survives while a 10-day-old uncurated fight is pruned. + """ + shortest = min(DETAIL_RETENTION_RAID_DAYS, DETAIL_RETENTION_DUNGEON_DAYS, DETAIL_RETENTION_OTHER_DAYS) + # One fetch: the candidate payload is three small columns and the partial + # index keeps the set to "kept fights whose shortest tier has passed but + # whose own tier may not have" — a few hundred rows in steady state. + rows = parses_db.select_detail_prune_candidates(conn, older_than=now - shortest * _DAY_S, limit=100_000) + due = [r["id"] for r in rows if r["started_at"] < now - detail_retention_days(r.get("zone")) * _DAY_S] + + pruned = 0 + for i in range(0, len(due), _PRUNE_CHUNK): + chunk = due[i : i + _PRUNE_CHUNK] + parses_db.prune_encounter_detail(conn, chunk, pruned_at=now) + conn.commit() + pruned += len(chunk) + return pruned diff --git a/scripts/dev/seed_fake_parses.py b/scripts/dev/seed_fake_parses.py index 9f36ed2e..547d7c54 100644 --- a/scripts/dev/seed_fake_parses.py +++ b/scripts/dev/seed_fake_parses.py @@ -32,6 +32,7 @@ import sys import time import uuid +from typing import Any sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) @@ -98,18 +99,18 @@ def _curated_bosses(limit: int) -> list[tuple[str, str]]: ][:limit] -def wipe(conn: sqlite3.Connection) -> None: +def wipe(conn: Any) -> None: n = conn.execute( - "DELETE FROM combatants WHERE encounter_id IN (SELECT id FROM encounters WHERE uploaded_by = ?)", + "DELETE FROM combatants WHERE encounter_id IN (SELECT id FROM encounters WHERE uploaded_by = %s)", (UPLOADER,), ).rowcount - m = conn.execute("DELETE FROM encounters WHERE uploaded_by = ?", (UPLOADER,)).rowcount + m = conn.execute("DELETE FROM encounters WHERE uploaded_by = %s", (UPLOADER,)).rowcount conn.commit() print(f"wiped {m} fake encounters ({n} combatant rows)") def _insert_kill( - conn: sqlite3.Connection, + conn: Any, *, world: str, zone: str, @@ -123,7 +124,7 @@ def _insert_kill( """INSERT INTO encounters (world, act_encid, title, zone, started_at, ended_at, duration_s, total_damage, encdps, kills, deaths, success_level, source_dsn, uploaded_by, guild_name, ingested_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 0, 1, ?, ?, ?, ?)""", + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 1, 0, 1, %s, %s, %s, %s) RETURNING id""", ( world, f"fake-{uuid.uuid4().hex[:12]}", @@ -140,13 +141,14 @@ def _insert_kill( int(time.time()), ), ) - enc_id = cur.lastrowid + _row = cur.fetchone() + enc_id = _row["id"] if _row else 0 for name, cls, dps, hps in combatants: conn.execute( """INSERT INTO combatants (encounter_id, name, ally, started_at, ended_at, duration_s, damage, dps, encdps, enchps, level, guild_name, cls, ilvl, is_player) - VALUES (?, ?, 1, ?, ?, ?, ?, ?, ?, ?, 70, ?, ?, ?, 1)""", + VALUES (%s, %s, 1, %s, %s, %s, %s, %s, %s, %s, 70, %s, %s, %s, 1)""", ( enc_id, name, @@ -164,7 +166,7 @@ def _insert_kill( ) -def seed(conn: sqlite3.Connection, *, character: str, cls: str, world: str) -> None: +def seed(conn: Any, *, character: str, cls: str, world: str) -> None: rng = random.Random(42) bosses = _curated_bosses(len(BOSS_SHAPES)) now = int(time.time()) diff --git a/scripts/dev/seed_raid_kills.py b/scripts/dev/seed_raid_kills.py index 1ca91c63..e9923494 100644 --- a/scripts/dev/seed_raid_kills.py +++ b/scripts/dev/seed_raid_kills.py @@ -22,10 +22,10 @@ import json import os import random -import sqlite3 import sys import time from pathlib import Path +from typing import Any # Repo root on path so the script can import as a module. _REPO = Path(__file__).resolve().parent.parent.parent @@ -37,11 +37,11 @@ from backend.census.client import CensusClient # noqa: E402 from backend.eq2db import zones as zones_db # noqa: E402 from backend.server.db import SCHEMA as USERS_SCHEMA # noqa: E402 -from backend.server.parses.db import DB_PATH as PARSES_DB_PATH # noqa: E402 +from backend.server.parses.db import store as parses_store # noqa: E402 load_dotenv(_REPO / ".env") -_MANIFEST = PARSES_DB_PATH.parent / ".seed_raid_kills.json" +_MANIFEST = _REPO / "data" / "parses" / ".seed_raid_kills.json" # --------------------------------------------------------------------------- # Seed plan — which encounters to kill in which zones. @@ -182,7 +182,7 @@ def _generate_rows(guild_name: str, discord_id: str) -> list[dict]: return rows -def _delete_previous(conn: sqlite3.Connection) -> int: +def _delete_previous(conn: Any) -> int: """Remove rows from the previous seed run, if any.""" if not _MANIFEST.exists(): return 0 @@ -192,8 +192,7 @@ def _delete_previous(conn: sqlite3.Connection) -> int: ids = [] if not ids: return 0 - placeholders = ",".join("?" * len(ids)) - cur = conn.execute(f"DELETE FROM encounters WHERE id IN ({placeholders})", ids) + cur = conn.execute("DELETE FROM encounters WHERE id = ANY(%s)", (list(ids),)) conn.commit() deleted = cur.rowcount _MANIFEST.unlink(missing_ok=True) @@ -216,13 +215,9 @@ async def main() -> int: parser.add_argument("--guild", help="Override the auto-detected guild name (e.g. for testing other guilds).") args = parser.parse_args() - if not PARSES_DB_PATH.exists(): - print(f"parses.db not found at {PARSES_DB_PATH} — nothing to seed/unseed.") - return 1 - # ─── Unseed-only path ───────────────────────────────────────────────────── if args.unseed: - with sqlite3.connect(PARSES_DB_PATH) as conn: + with parses_store.init_db() as conn: n = _delete_previous(conn) print(f"Removed {n} previously-seeded rows.") return 0 @@ -259,7 +254,7 @@ async def main() -> int: print("Nothing to insert (seed plan resolved to zero rows).") return 1 - with sqlite3.connect(PARSES_DB_PATH) as conn: + with parses_store.init_db() as conn: cleared = _delete_previous(conn) inserted_ids: list[int] = [] for r in rows: @@ -269,13 +264,15 @@ async def main() -> int: (act_encid, title, zone, started_at, ended_at, duration_s, success_level, source_dsn, uploaded_by, guild_name, ingested_at) VALUES - (:act_encid, :title, :zone, :started_at, :ended_at, :duration_s, - :success_level, :source_dsn, :uploaded_by, :guild_name, :ingested_at) + (%(act_encid)s, %(title)s, %(zone)s, %(started_at)s, %(ended_at)s, %(duration_s)s, + %(success_level)s, %(source_dsn)s, %(uploaded_by)s, %(guild_name)s, %(ingested_at)s) + RETURNING id """, r, ) - if cur.lastrowid is not None: - inserted_ids.append(int(cur.lastrowid)) + _row = cur.fetchone() + if _row is not None: + inserted_ids.append(int(_row["id"])) conn.commit() _write_manifest(inserted_ids) diff --git a/tests/conftest.py b/tests/conftest.py index bb8b1367..2a8f398a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,7 +65,6 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 BE-096: moved from module-level os.environ calls to avoid a race with pytest plugins (e.g. pytest-asyncio) that may import web.app during plugin discovery.""" - os.environ["DB_PARSES_PATH"] = str(_TEST_DB_DIR / "backend.server.parses.db") os.environ["DB_CENSUS_PATH"] = str(_TEST_DB_DIR / "backend.census.db") os.environ["DB_ZONES_PATH"] = str(_TEST_DB_DIR / "zones.db") os.environ["DB_SPELLS_PATH"] = str(_TEST_DB_DIR / "spells.db") @@ -121,10 +120,7 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 from backend.eq2db import recipes as recipes_db from backend.eq2db import spells as spells_db from backend.eq2db import zones as zones_db - from backend.server.parses import db as parses_db - parses_db.DB_PATH = resolve_db_path("DB_PARSES_PATH", "parses", "parses.db") - parses_db.store.path = parses_db.DB_PATH census_store.DB_PATH = resolve_db_path("DB_CENSUS_PATH", "census", "census.db") census_store.store.path = census_store.DB_PATH # eq2db catalogue modules: re-point both the module constant AND the @@ -140,11 +136,9 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 mod.DB_PATH = resolve_db_path(env_var, subdir, filename) mod.catalogue.path = mod.DB_PATH - # Create the parses schema immediately. FastAPI's startup hooks don't - # fire under ASGITransport, so without this step parses tests would hit - # a missing-table OperationalError the first time they read from the DB. - # (The users family is provisioned by provision_for_session above.) - parses_db.store.init_db() + # (The users + parses schemas are provisioned by provision_for_session + # above — FastAPI's startup hooks don't fire under ASGITransport, but + # the migrations already ran against the test database.) from unittest.mock import AsyncMock, MagicMock # noqa: E402 diff --git a/tests/fixtures/parses_db.py b/tests/fixtures/parses_db.py index 8638bc33..6a7eea6d 100644 --- a/tests/fixtures/parses_db.py +++ b/tests/fixtures/parses_db.py @@ -1,49 +1,51 @@ -"""Shared parses-DB fixture used by tests/parses/ AND tests/web/. +"""Shared parses fixtures used by tests/parses/ AND tests/server/. -Hoisted from tests/parses/conftest.py per TEST-011: 7+ tests in -tests/web/test_parses_world_scoping.py were re-doing the same -monkeypatch + init_db setup inline. +Postgres edition: the parses family lives in the ``parses`` schema +(db/migrations/0002_parses.sql). Two fixtures are exposed, keeping their +historical names so test bodies don't churn: -Two fixtures are exposed: + - parses_db_path: leases an isolated scratch parses schema, points the + shared store at it for the test, and yields the SCHEMA NAME (str) — + the analog of the old tmp-file + DB_PATH re-point. - - parses_db_path: yields a tmp_path / "backend.server.parses.db" with the schema - pre-initialised, AND re-points parses_db.DB_PATH + the shared store at it - for the duration of the test. This is what the web tests need. - - - parses_db_conn: yields an in-memory connection, the schema applied. - Matches the existing tests/parses/conftest.py fixture name — kept - for parses unit tests that need a connection handle. + - parses_db_conn: leases a scratch schema and yields an open, + schema-scoped connection (PgConnProxy — dict rows, %s params); the + store's conn-taking staticmethods accept it directly. """ from __future__ import annotations -import sqlite3 from collections.abc import Generator -from pathlib import Path +from typing import Any import pytest from backend.server.parses import db as parses_db +from tests.fixtures.pg import SchemaLeaser + +#: Scratch-schema pool for the parses family (parses_s_N). +parses_leaser = SchemaLeaser("parses", "0002_parses.sql") @pytest.fixture -def parses_db_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Tmp-path-backed parses.db with schema initialised + store re-pointed.""" - db_file = tmp_path / "backend.server.parses.db" - # Re-point BOTH the module constant (metrics.py reads it dynamically) - # and the shared store instance (what the routes read). - monkeypatch.setattr(parses_db, "DB_PATH", db_file) - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - return db_file +def parses_db_path(monkeypatch: pytest.MonkeyPatch) -> Generator[str]: + """Isolated leased parses schema with the shared store re-pointed.""" + name = parses_leaser.acquire() + monkeypatch.setattr(parses_db, "SCHEMA", name) + monkeypatch.setattr(parses_db.store, "schema", name) + try: + yield name + finally: + parses_leaser.release(name) @pytest.fixture -def parses_db_conn(tmp_path: Path) -> Generator[sqlite3.Connection]: - """Throwaway parses DB connection (schema already applied). - - File-backed (BaseCatalogue dropped :memory: support — per-read - connections would each see a fresh empty memory DB).""" - conn = parses_db.ParsesStore(tmp_path / "conn_parses.db").init_db() - yield conn - conn.close() +def parses_db_conn() -> Generator[Any]: + """Throwaway schema-scoped parses connection (leased scratch schema).""" + name = parses_leaser.acquire() + conn = parses_db.ParsesStore(name).init_db() + try: + yield conn + finally: + conn.close() + parses_leaser.release(name) From 63227152b15a214d1375bf7314673c0b32951b19 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 16:31:12 +0100 Subject: [PATCH 19/32] feat(pg): census family on psycopg + parses test suite converted; detail-pruned notice 0003_census.sql: reviewed census schema (jsonb data_json, text_pattern_ops prefix indexes, DISTINCT ON latest-member-counts index). CensusStore on PgCatalogue: Json() writes, parsed jsonb reads (per-row json.loads gone, cls extracted in SQL), DISTINCT ON replaces the bare-column-MAX idiom; path guards dropped at the five census consumers. census_schema leased fixture + conftest rewiring. Erasure reads the parses schema at call time (test-pointable symmetry). All 21 parses test files converted by three agents + verified: schema-shape tests became information_schema invariants, ingest's shared-connection tricks dropped (READ COMMITTED), delete's ANY() wire shape asserted. New tests/server/parses/ test_detail_retention.py pins the 30/14/7 tier sweep. ParsePage shows a "breakdown pruned" notice when detail_pruned_at is set (450/450 vitests). Co-Authored-By: Claude Fable 5 --- backend/census/store.py | 137 +++--- backend/census/store.sql | 119 +---- backend/server/api/characters.py | 2 - backend/server/api/guild.py | 2 - backend/server/api/guild_recruitment.py | 2 +- backend/server/api/refresh.py | 4 - backend/server/db/erasure.py | 4 +- backend/server/metrics.py | 2 - db/migrations/0003_census.sql | 77 +++ frontend/src/pages/ParsePage.tsx | 9 + tests/bot/test_parse_posts.py | 21 +- tests/conftest.py | 5 +- tests/fixtures/census_db.py | 31 ++ tests/server/parses/test_cleanup.py | 29 +- tests/server/parses/test_db.py | 437 ++++++------------ tests/server/parses/test_db_is_player.py | 127 +++-- tests/server/parses/test_detail_retention.py | 114 +++++ tests/server/test_admin_tamper_reports.py | 32 +- tests/server/test_attendance.py | 13 +- tests/server/test_erasure.py | 65 ++- tests/server/test_export.py | 19 +- tests/server/test_parses_delete.py | 17 +- tests/server/test_parses_ingest.py | 14 +- tests/server/test_parses_ingest_classify.py | 101 ++-- .../test_parses_ingest_client_warnings.py | 49 +- tests/server/test_parses_ingest_duplicates.py | 39 +- tests/server/test_parses_list.py | 20 +- tests/server/test_parses_pagination.py | 21 +- tests/server/test_parses_pet_filter.py | 119 +++-- tests/server/test_parses_tamper_report.py | 18 +- tests/server/test_parses_world_scoping.py | 96 +--- tests/server/test_rankings.py | 32 +- tests/server/test_zones_cache_invalidation.py | 45 +- 33 files changed, 829 insertions(+), 993 deletions(-) create mode 100644 db/migrations/0003_census.sql create mode 100644 tests/fixtures/census_db.py create mode 100644 tests/server/parses/test_detail_retention.py diff --git a/backend/census/store.py b/backend/census/store.py index fc1058fb..20bffcb3 100644 --- a/backend/census/store.py +++ b/backend/census/store.py @@ -1,36 +1,36 @@ """Persistent, deploy-surviving store of the last-known character + guild -lookups. The web request path serves from here (via the in-memory cache) and -never blocks on Census; background refreshes merge in fresh data "keep best -known" — a sparse Census response never nulls out good data. +lookups (the ``census`` Postgres schema). The web request path serves from +here (via the in-memory cache) and never blocks on Census; background +refreshes merge in fresh data "keep best known" — a sparse Census response +never nulls out good data. All behaviour lives on :class:`CensusStore` (the catalogue convention — see backend/db_catalogue.py): the shared module-level ``store`` instance is the runtime entry point (consumers alias it ``census_store``); the get/upsert helpers take an open conn (callers batch reads/writes per connection) and are -staticmethods. Tests construct ``CensusStore(tmp_db)``. SQL lives in the sibling -store.sql (schema_* + DML blocks). +staticmethods. ``init_db()`` returns a pooled schema-scoped connection proxy; +schema DDL lives in db/migrations/0003_census.sql. ``data_json`` is jsonb — +psycopg hands back parsed dicts and writes go through ``Json()``. """ from __future__ import annotations -import json import logging -import sqlite3 import time from collections.abc import Iterable from datetime import UTC, datetime -from pathlib import Path from typing import Any, TypedDict -from backend.db_catalogue import BaseCatalogue -from backend.db_helpers import resolve_db_path +from psycopg.types.json import Json + +from backend.db_catalogue import PgCatalogue from backend.sql_loader import load_sql class StoreRecord(TypedDict): """Envelope returned by ``get_character`` / ``get_guild`` / ``get_character_aas``. - ``data`` is the original model_dump() dict stored as JSON; the caller + ``data`` is the original model_dump() dict stored as jsonb; the caller deserialises field-by-field as needed. ``last_resolved_at`` is a Unix timestamp of when Census last responded successfully for this entity. """ @@ -60,41 +60,23 @@ class GuildHistoryPoint(GuildHistorySnapshot): _log = logging.getLogger(__name__) - -DB_PATH: Path = resolve_db_path("DB_CENSUS_PATH", "census", "census.db") - _SQL = load_sql(__file__) -_MIGRATIONS: list[str] = [] # future schema bumps appended here - +#: Postgres schema the census family lives in (db/migrations/0003_census.sql). +SCHEMA = "census" -class CensusStore(BaseCatalogue): - """Read/write access to one census.db file (last-known Census lookups).""" - # The three tables declare no foreign keys — the old init set the pragma - # as boilerplate, not because anything relied on cascade. - FOREIGN_KEYS = False +class CensusStore(PgCatalogue): + """Read/write access to the census schema (last-known Census lookups).""" - # census.db predates the shared _meta table and has no build provenance - # to track — rows carry their own timestamps. - CREATE_META = False - - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) - - def _create_schema(self, conn: sqlite3.Connection) -> None: - conn.execute(_SQL["schema_characters"]) - conn.execute(_SQL["schema_guilds"]) - conn.execute(_SQL["schema_character_aas"]) - conn.execute(_SQL["schema_character_gear_sets"]) - conn.execute(_SQL["schema_guild_history"]) - self._apply_migrations(conn, _MIGRATIONS) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) # ── Characters ─────────────────────────────────────────────────────────── @staticmethod def upsert_character( - conn: sqlite3.Connection, + conn: Any, name: str, world: str, data: dict, @@ -135,7 +117,7 @@ def upsert_character( name, data.get("level"), data.get("guild_name"), - json.dumps(data), + Json(data), resolved_ts, write_ts, ), @@ -143,12 +125,12 @@ def upsert_character( conn.commit() @staticmethod - def get_character(conn: sqlite3.Connection, name: str, world: str) -> StoreRecord | None: + def get_character(conn: Any, name: str, world: str) -> StoreRecord | None: """Return {data, last_resolved_at} or None.""" row = conn.execute(_SQL["select_character"], (name.lower(), world)).fetchone() if row is None: return None - return {"data": json.loads(row[0]), "last_resolved_at": row[1]} + return {"data": row["data_json"], "last_resolved_at": row["last_resolved_at"]} @staticmethod def _like_prefix(prefix: str) -> str: @@ -156,49 +138,43 @@ def _like_prefix(prefix: str) -> str: return prefix.lower().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" @staticmethod - def search_characters(conn: sqlite3.Connection, prefix: str, world: str, limit: int = 20) -> list[dict]: + def search_characters(conn: Any, prefix: str, world: str, limit: int = 20) -> list[dict]: """Name-prefix search over every character this server has seen (guild-roster merges pull whole guilds in) — the instant half of - /characters/search. Returns [{name, level, guild_name, cls}].""" + /characters/search. Returns [{name, level, guild_name, cls}]. + ``cls`` comes straight from the jsonb blob in SQL.""" rows = conn.execute( _SQL["search_characters_by_prefix"], (world, CensusStore._like_prefix(prefix), limit) ).fetchall() - out = [] - for name, level, guild_name, data_json in rows: - try: - cls = (json.loads(data_json) or {}).get("cls") - except (TypeError, ValueError): - cls = None - out.append({"name": name, "level": level, "guild_name": guild_name, "cls": cls}) - return out + return [{"name": r["name"], "level": r["level"], "guild_name": r["guild_name"], "cls": r["cls"]} for r in rows] @staticmethod - def search_guilds(conn: sqlite3.Connection, prefix: str, world: str, limit: int = 20) -> list[str]: + def search_guilds(conn: Any, prefix: str, world: str, limit: int = 20) -> list[str]: """Name-prefix search over every guild this server has seen.""" rows = conn.execute( _SQL["search_guilds_by_prefix"], (world, CensusStore._like_prefix(prefix), limit) ).fetchall() - return [r[0] for r in rows] + return [r["name"] for r in rows] # ── Guilds ─────────────────────────────────────────────────────────────── @staticmethod - def upsert_guild(conn: sqlite3.Connection, name: str, world: str, data: dict, *, now: int | None = None) -> None: + def upsert_guild(conn: Any, name: str, world: str, data: dict, *, now: int | None = None) -> None: """Store the guild roster blob (member names+ranks + info). Always replaces — the roster list is reliable from Census regardless of member login recency.""" ts = int(time.time()) if now is None else now conn.execute( _SQL["upsert_guild"], - (name.lower(), world, name, json.dumps(data), ts, ts), + (name.lower(), world, name, Json(data), ts, ts), ) conn.commit() @staticmethod - def get_guild(conn: sqlite3.Connection, name: str, world: str) -> StoreRecord | None: + def get_guild(conn: Any, name: str, world: str) -> StoreRecord | None: row = conn.execute(_SQL["select_guild"], (name.lower(), world)).fetchone() if row is None: return None - return {"data": json.loads(row[0]), "last_resolved_at": row[1]} + return {"data": row["data_json"], "last_resolved_at": row["last_resolved_at"]} # ── Guild history ──────────────────────────────────────────────────────── @@ -209,7 +185,7 @@ def _utc_day(ts: int) -> str: @staticmethod def upsert_guild_history( - conn: sqlite3.Connection, + conn: Any, name: str, world: str, snapshot: GuildHistorySnapshot, @@ -241,7 +217,7 @@ def upsert_guild_history( @staticmethod def get_guild_history( - conn: sqlite3.Connection, name: str, world: str, days: int, *, now: int | None = None + conn: Any, name: str, world: str, days: int, *, now: int | None = None ) -> list[GuildHistoryPoint]: """The guild's daily rows from ``days`` days ago (UTC) to today, oldest first. Empty list for a guild with no history.""" @@ -250,37 +226,37 @@ def get_guild_history( rows = conn.execute(_SQL["select_guild_history"], (world, name.lower(), since)).fetchall() return [ { - "day": r[0], - "captured_at": r[1], - "level": r[2], - "members": r[3], - "accounts": r[4], - "achievement_count": r[5], - "max_level_members": r[6], - "distinct_classes": r[7], + "day": r["day"], + "captured_at": r["captured_at"], + "level": r["level"], + "members": r["members"], + "accounts": r["accounts"], + "achievement_count": r["achievement_count"], + "max_level_members": r["max_level_members"], + "distinct_classes": r["distinct_classes"], } for r in rows ] @staticmethod - def latest_guild_member_counts(conn: sqlite3.Connection, world: str, names_lower: Iterable[str]) -> dict[str, int]: + def latest_guild_member_counts(conn: Any, world: str, names_lower: Iterable[str]) -> dict[str, int]: """Latest-known member count for each requested guild, keyed by name_lower; guilds with no history rows are simply absent. One - grouped pass over guild_history (bare ``members`` rides the - MAX(day) row). Feeds the recruiting browse cards.""" + DISTINCT ON pass over guild_history. Feeds the recruiting browse + cards.""" wanted = set(names_lower) if not wanted: return {} out: dict[str, int] = {} - for name_lower, members, _day in conn.execute(_SQL["select_latest_member_counts"], (world,)): - if name_lower in wanted and members is not None: - out[name_lower] = int(members) + for r in conn.execute(_SQL["select_latest_member_counts"], (world,)).fetchall(): + if r["name_lower"] in wanted and r["members"] is not None: + out[r["name_lower"]] = int(r["members"]) return out # ── Character AAs ──────────────────────────────────────────────────────── @staticmethod - def get_character_aas(conn: sqlite3.Connection, name: str, world: str) -> StoreRecord | None: + def get_character_aas(conn: Any, name: str, world: str) -> StoreRecord | None: """Return the persisted CharAAsResponse dict (or None) for (name, world). The record carries the model_dump() of the response plus a @@ -288,14 +264,11 @@ def get_character_aas(conn: sqlite3.Connection, name: str, world: str) -> StoreR row = conn.execute(_SQL["select_character_aas"], (name.lower(), world)).fetchone() if row is None: return None - return { - "data": json.loads(row[0]), - "last_resolved_at": row[1], - } + return {"data": row["data_json"], "last_resolved_at": row["last_resolved_at"]} @staticmethod def upsert_character_aas( - conn: sqlite3.Connection, + conn: Any, name: str, world: str, data: dict, @@ -307,22 +280,22 @@ def upsert_character_aas( authoritative.""" if now is None: now = int(time.time()) - conn.execute(_SQL["upsert_character_aas"], (name.lower(), world, json.dumps(data), now)) + conn.execute(_SQL["upsert_character_aas"], (name.lower(), world, Json(data), now)) conn.commit() # ── Character gear sets ────────────────────────────────────────────────── @staticmethod - def get_character_gear_sets(conn: sqlite3.Connection, name: str, world: str) -> StoreRecord | None: + def get_character_gear_sets(conn: Any, name: str, world: str) -> StoreRecord | None: """The persisted gear-sets response dict (or None) for (name, world).""" row = conn.execute(_SQL["select_character_gear_sets"], (name.lower(), world)).fetchone() if row is None: return None - return {"data": json.loads(row[0]), "last_resolved_at": row[1]} + return {"data": row["data_json"], "last_resolved_at": row["last_resolved_at"]} @staticmethod def upsert_character_gear_sets( - conn: sqlite3.Connection, + conn: Any, name: str, world: str, data: dict, @@ -336,7 +309,7 @@ def upsert_character_gear_sets( now = int(time.time()) conn.execute( _SQL["upsert_character_gear_sets"], - (name.lower(), world, json.dumps(data), now), + (name.lower(), world, Json(data), now), ) conn.commit() diff --git a/backend/census/store.sql b/backend/census/store.sql index eeebaec8..0b3c38f2 100644 --- a/backend/census/store.sql +++ b/backend/census/store.sql @@ -1,148 +1,79 @@ --- SQL for backend/census/store.py (CensusStore). Schema + DML both live --- here per the sql_loader convention — one grep target for all SQL. - --- --------------------------------------------------------------------------- --- Schema --- --------------------------------------------------------------------------- - --- :name schema_characters -CREATE TABLE IF NOT EXISTS characters ( - name_lower TEXT NOT NULL, - world TEXT NOT NULL, - name TEXT NOT NULL, - level INTEGER, - guild_name TEXT, - data_json TEXT NOT NULL, - last_resolved_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (name_lower, world) -); - --- :name schema_guilds -CREATE TABLE IF NOT EXISTS guilds ( - name_lower TEXT NOT NULL, - world TEXT NOT NULL, - name TEXT NOT NULL, - data_json TEXT NOT NULL, - last_resolved_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (name_lower, world) -); - --- :name schema_character_aas -CREATE TABLE IF NOT EXISTS character_aas ( - name_lower TEXT NOT NULL, - world TEXT NOT NULL, - data_json TEXT NOT NULL, - last_resolved_at INTEGER NOT NULL, - PRIMARY KEY (name_lower, world) -); - --- :name schema_character_gear_sets -CREATE TABLE IF NOT EXISTS character_gear_sets ( - name_lower TEXT NOT NULL, - world TEXT NOT NULL, - data_json TEXT NOT NULL, - last_resolved_at INTEGER NOT NULL, - PRIMARY KEY (name_lower, world) -); - --- One row per guild per UTC day: the headline numbers of a guild refresh --- (level / members / accounts / achievements plus two roster reductions). --- Feeds the guild page History charts. day is 'YYYY-MM-DD' (UTC); the --- 15-minute refresh overwrites the same day's row so the last capture of --- the day wins. max_level_members is NULL when the world's registry row --- (and so its max level) was unknown at capture time. --- :name schema_guild_history -CREATE TABLE IF NOT EXISTS guild_history ( - world TEXT NOT NULL, - name_lower TEXT NOT NULL, - day TEXT NOT NULL, - captured_at INTEGER NOT NULL, - level INTEGER, - members INTEGER, - accounts INTEGER, - achievement_count INTEGER, - max_level_members INTEGER, - distinct_classes INTEGER, - PRIMARY KEY (world, name_lower, day) -); - --- --------------------------------------------------------------------------- --- DML --- --------------------------------------------------------------------------- +-- SQL for backend/census/store.py (psycopg, census schema). DML only — +-- schema DDL lives in db/migrations/0003_census.sql. -- :name upsert_guild_history INSERT INTO guild_history (world, name_lower, day, captured_at, level, members, accounts, achievement_count, max_level_members, distinct_classes) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT(world, name_lower, day) DO UPDATE SET captured_at=excluded.captured_at, level=excluded.level, members=excluded.members, accounts=excluded.accounts, achievement_count=excluded.achievement_count, max_level_members=excluded.max_level_members, distinct_classes=excluded.distinct_classes; -- :name prune_guild_history -DELETE FROM guild_history WHERE world = ? AND name_lower = ? AND day < ?; +DELETE FROM guild_history WHERE world = %s AND name_lower = %s AND day < %s; -- :name select_guild_history SELECT day, captured_at, level, members, accounts, achievement_count, max_level_members, distinct_classes FROM guild_history -WHERE world = ? AND name_lower = ? AND day >= ? +WHERE world = %s AND name_lower = %s AND day >= %s ORDER BY day; -- Latest-known member count per guild on a world — one row per guild from --- its most recent history day (SQLite's bare-column-with-MAX rule pins --- members to the MAX(day) row). Feeds the recruiting browse cards. +-- its most recent history day. DISTINCT ON replaces SQLite's bare-column- +-- with-MAX idiom (a hard error on Postgres); idx_guild_history_latest +-- makes it an ordered index scan. -- :name select_latest_member_counts -SELECT name_lower, members, MAX(day) AS day +SELECT DISTINCT ON (name_lower) name_lower, members, day FROM guild_history -WHERE world = ? -GROUP BY name_lower; +WHERE world = %s +ORDER BY name_lower, day DESC; -- :name upsert_character INSERT INTO characters (name_lower, world, name, level, guild_name, data_json, last_resolved_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT(name_lower, world) DO UPDATE SET name=excluded.name, level=excluded.level, guild_name=excluded.guild_name, data_json=excluded.data_json, last_resolved_at=excluded.last_resolved_at, updated_at=excluded.updated_at; -- :name select_character -SELECT data_json, last_resolved_at FROM characters WHERE name_lower=? AND world=?; +SELECT data_json, last_resolved_at FROM characters WHERE name_lower=%s AND world=%s; -- :name upsert_guild INSERT INTO guilds (name_lower, world, name, data_json, last_resolved_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?) +VALUES (%s, %s, %s, %s, %s, %s) ON CONFLICT(name_lower, world) DO UPDATE SET name=excluded.name, data_json=excluded.data_json, last_resolved_at=excluded.last_resolved_at, updated_at=excluded.updated_at; -- :name select_guild -SELECT data_json, last_resolved_at FROM guilds WHERE name_lower=? AND world=?; +SELECT data_json, last_resolved_at FROM guilds WHERE name_lower=%s AND world=%s; -- :name select_character_aas -SELECT data_json, last_resolved_at FROM character_aas WHERE name_lower = ? AND world = ?; +SELECT data_json, last_resolved_at FROM character_aas WHERE name_lower = %s AND world = %s; -- :name upsert_character_aas -INSERT INTO character_aas (name_lower, world, data_json, last_resolved_at) VALUES (?, ?, ?, ?) +INSERT INTO character_aas (name_lower, world, data_json, last_resolved_at) VALUES (%s, %s, %s, %s) ON CONFLICT(name_lower, world) DO UPDATE SET data_json = excluded.data_json, last_resolved_at = excluded.last_resolved_at; -- :name select_character_gear_sets -SELECT data_json, last_resolved_at FROM character_gear_sets WHERE name_lower = ? AND world = ?; +SELECT data_json, last_resolved_at FROM character_gear_sets WHERE name_lower = %s AND world = %s; -- :name upsert_character_gear_sets -INSERT INTO character_gear_sets (name_lower, world, data_json, last_resolved_at) VALUES (?, ?, ?, ?) +INSERT INTO character_gear_sets (name_lower, world, data_json, last_resolved_at) VALUES (%s, %s, %s, %s) ON CONFLICT(name_lower, world) DO UPDATE SET data_json = excluded.data_json, last_resolved_at = excluded.last_resolved_at; -- Name-prefix search over everything this server has ever seen — the -- store-first half of /characters/search and /guilds/search (the census -- round-trip stays off the keystroke path). Callers escape LIKE wildcards. +-- cls is pulled from the jsonb blob in SQL (was a per-row json.loads). -- :name search_characters_by_prefix -SELECT name, level, guild_name, data_json FROM characters -WHERE world = ? AND name_lower LIKE ? ESCAPE '\' -ORDER BY name_lower LIMIT ?; +SELECT name, level, guild_name, data_json->>'cls' AS cls FROM characters +WHERE world = %s AND name_lower LIKE %s ESCAPE '\' +ORDER BY name_lower LIMIT %s; -- :name search_guilds_by_prefix SELECT name FROM guilds -WHERE world = ? AND name_lower LIKE ? ESCAPE '\' -ORDER BY name_lower LIMIT ?; +WHERE world = %s AND name_lower LIKE %s ESCAPE '\' +ORDER BY name_lower LIMIT %s; diff --git a/backend/server/api/characters.py b/backend/server/api/characters.py index 7e0f4a18..44632f05 100644 --- a/backend/server/api/characters.py +++ b/backend/server/api/characters.py @@ -54,8 +54,6 @@ class CharSearchResponse(BaseModel): def _store_search_sync(q: str, world: str) -> list[CharNameResult]: """SYNC (executor): census-store name-prefix search.""" - if not census_store.path.exists(): - return [] conn = census_store.init_db() try: rows = census_store.search_characters(conn, q, world) diff --git a/backend/server/api/guild.py b/backend/server/api/guild.py index c4166e01..248f87cd 100644 --- a/backend/server/api/guild.py +++ b/backend/server/api/guild.py @@ -501,8 +501,6 @@ async def search_guilds(name: str = "") -> GuildSearchResponse: world = current_world() def _store_guild_search() -> list[str]: - if not census_store.path.exists(): - return [] conn = census_store.init_db() try: return census_store.search_guilds(conn, q, world) diff --git a/backend/server/api/guild_recruitment.py b/backend/server/api/guild_recruitment.py index 48033694..fc746d38 100644 --- a/backend/server/api/guild_recruitment.py +++ b/backend/server/api/guild_recruitment.py @@ -449,7 +449,7 @@ async def list_recruiting(request: Request) -> RecruitingListResponse: rows = await recruitment_db.list_recruiting(world) counts: dict[str, int] = {} - if rows and census_store.path.exists(): + if rows: def _read_counts() -> dict[str, int]: conn = census_store.init_db() diff --git a/backend/server/api/refresh.py b/backend/server/api/refresh.py index 9a3db4ea..9143180c 100644 --- a/backend/server/api/refresh.py +++ b/backend/server/api/refresh.py @@ -34,8 +34,6 @@ def _last_resolved_character(name: str, world: str) -> int | None: - if not census_store.path.exists(): - return None conn = census_store.init_db() try: rec = census_store.get_character(conn, name, world) @@ -45,8 +43,6 @@ def _last_resolved_character(name: str, world: str) -> int | None: def _last_resolved_guild(name: str, world: str) -> int | None: - if not census_store.path.exists(): - return None conn = census_store.init_db() try: rec = census_store.get_guild(conn, name, world) diff --git a/backend/server/db/erasure.py b/backend/server/db/erasure.py index 277c9451..20168ec6 100644 --- a/backend/server/db/erasure.py +++ b/backend/server/db/erasure.py @@ -34,7 +34,7 @@ from backend import pg from backend.server import db as _users_db # SCHEMA read at call time (tests re-point it) -from backend.server.parses.db import SCHEMA as _PARSES_SCHEMA +from backend.server.parses import db as _parses_db # SCHEMA read at call time (tests re-point it) #: The placeholder users row every tombstoned reference points at. DELETED_USER_ID = "deleted" @@ -164,7 +164,7 @@ def erase_user_sync( # SET search_path is transactional, so the whole erasure commits or # rolls back as one unit — the old users.db/parses.db split could # crash between the halves and leave a half-erased account. - conn.execute(pg.search_path_sql(parses_schema if parses_schema is not None else _PARSES_SCHEMA)) + conn.execute(pg.search_path_sql(parses_schema if parses_schema is not None else _parses_db.SCHEMA)) dsn = f"plugin:{discord_id}" cur = conn.execute("UPDATE encounters SET source_dsn = %s WHERE source_dsn = %s", (DELETED_SOURCE_DSN, dsn)) result.parses_anonymised = cur.rowcount diff --git a/backend/server/metrics.py b/backend/server/metrics.py index c3a1dfe9..1e30ff16 100644 --- a/backend/server/metrics.py +++ b/backend/server/metrics.py @@ -336,7 +336,6 @@ class _DBFileSizeCollector(Collector): spot the difference via the labelset gap.""" def collect(self): # type: ignore[override] - from backend.census import store as census_store from backend.eq2db import classes as classes_db from backend.eq2db import items as items_db from backend.eq2db import raids as raids_db @@ -348,7 +347,6 @@ def collect(self): # type: ignore[override] # (users + parses moved to Postgres — covered by the schema-size # gauge below instead of a file stat.) candidates = [ - ("census", census_store.DB_PATH), ("raids", raids_db.DB_PATH), ("zones", zones_db.DB_PATH), ("items", items_db.DB_PATH), diff --git a/db/migrations/0003_census.sql b/db/migrations/0003_census.sql new file mode 100644 index 00000000..e4176963 --- /dev/null +++ b/db/migrations/0003_census.sql @@ -0,0 +1,77 @@ +create schema if not exists census; +set search_path to census, public; + +-- Reviewed translation of backend/census/store.sql's schema blocks. +-- Policy: epoch bigints, counters bigint, composite text PKs kept. +-- REVIEW: data_json becomes jsonb — psycopg returns it parsed (the +-- json.loads round-trips in store.py go away) and the character prefix +-- search reads ->>'cls' in SQL instead of parsing whole blobs in Python. + +CREATE TABLE characters ( + name_lower text NOT NULL, + world text NOT NULL, + name text NOT NULL, + level integer, + guild_name text, + data_json jsonb NOT NULL, + last_resolved_at bigint NOT NULL, + updated_at bigint NOT NULL, + PRIMARY KEY (name_lower, world) +); + +CREATE TABLE guilds ( + name_lower text NOT NULL, + world text NOT NULL, + name text NOT NULL, + data_json jsonb NOT NULL, + last_resolved_at bigint NOT NULL, + updated_at bigint NOT NULL, + PRIMARY KEY (name_lower, world) +); + +CREATE TABLE character_aas ( + name_lower text NOT NULL, + world text NOT NULL, + data_json jsonb NOT NULL, + last_resolved_at bigint NOT NULL, + PRIMARY KEY (name_lower, world) +); + +CREATE TABLE character_gear_sets ( + name_lower text NOT NULL, + world text NOT NULL, + data_json jsonb NOT NULL, + last_resolved_at bigint NOT NULL, + PRIMARY KEY (name_lower, world) +); + +-- One row per guild per UTC day: the headline numbers of a guild refresh +-- (level / members / accounts / achievements plus two roster reductions). +-- Feeds the guild page History charts. day is 'YYYY-MM-DD' (UTC); the +-- 15-minute refresh overwrites the same day's row so the last capture of +-- the day wins. max_level_members is NULL when the world's registry row +-- (and so its max level) was unknown at capture time. +CREATE TABLE guild_history ( + world text NOT NULL, + name_lower text NOT NULL, + day text NOT NULL, + captured_at bigint NOT NULL, + level integer, + members integer, + accounts integer, + achievement_count integer, + max_level_members integer, + distinct_classes integer, + PRIMARY KEY (world, name_lower, day) +); + +-- REVIEW: the store-first name-prefix searches (LIKE 'ab%' on name_lower) +-- need text_pattern_ops to use an index under non-C collations. +CREATE INDEX idx_characters_prefix ON characters (world, name_lower text_pattern_ops); +CREATE INDEX idx_guilds_prefix ON guilds (world, name_lower text_pattern_ops); +-- The recruiting browse's latest-member-count DISTINCT ON scan. +CREATE INDEX idx_guild_history_latest ON guild_history (world, name_lower, day DESC); + +-- seeds +-- (none — census is a runtime mirror; marker kept for the scratch-schema +-- leaser's uniform reset contract) diff --git a/frontend/src/pages/ParsePage.tsx b/frontend/src/pages/ParsePage.tsx index f13517da..051e511f 100644 --- a/frontend/src/pages/ParsePage.tsx +++ b/frontend/src/pages/ParsePage.tsx @@ -80,6 +80,9 @@ interface ParseDetail { uploaded_by: string uploader_discord_id: string | null uploader_display_name: string | null + // Set when the retention sweep dropped the per-ability breakdown rows + // (summary numbers above are kept forever) — see parses/cleanup.py. + detail_pruned_at?: number | null combatants: CombatantSummary[] } @@ -197,6 +200,12 @@ export default function ParsePage() { This parse has been removed from the parses list, but is preserved here because it holds a ranking.

)} + {data.detail_pruned_at != null && ( +

+ The per-ability breakdown for this fight has been pruned by the retention policy — summary numbers, + rankings and the combatant list are kept forever. +

+ )} {allies.length > 0 && ( )} diff --git a/tests/bot/test_parse_posts.py b/tests/bot/test_parse_posts.py index da467fa5..3dfd308a 100644 --- a/tests/bot/test_parse_posts.py +++ b/tests/bot/test_parse_posts.py @@ -7,7 +7,7 @@ from __future__ import annotations -import sqlite3 +from typing import Any import pytest @@ -19,17 +19,16 @@ @pytest.fixture -def seeded_db(tmp_path, monkeypatch): - """Temp parses DB wired into the module store (what collect reads).""" - db_path = tmp_path / "parses.db" - monkeypatch.setattr(parses_db.store, "path", db_path) - conn = parses_db.ParsesStore(db_path).init_db() +def seeded_db(parses_db_path): + """Open connection to a leased scratch parses schema wired into the + module store (what collect reads).""" + conn = parses_db.store.init_db() yield conn conn.close() def _insert_fight( - conn: sqlite3.Connection, + conn: Any, *, title: str = "Trakanon", started_at: int = NOW, @@ -44,7 +43,7 @@ def _insert_fight( cur = conn.execute( "INSERT INTO encounters (world, act_encid, title, zone, started_at, ended_at, duration_s, " "success_level, source_dsn, uploaded_by, guild_name, ingested_at) " - "VALUES (?, ?, ?, 'Trakanon''s Lair', ?, ?, ?, ?, 'eq2act', ?, ?, ?)", + "VALUES (%s, %s, %s, 'Trakanon''s Lair', %s, %s, %s, %s, 'eq2act', %s, %s, %s) RETURNING id", ( world, f"enc-{started_at}-{uploaded_by}", @@ -58,14 +57,14 @@ def _insert_fight( ingested_at if ingested_at is not None else started_at + duration_s, ), ) - enc_id = int(cur.lastrowid or 0) + enc_id = int(cur.fetchone()["id"]) # is_player set explicitly so classification never reruns in tests; the # same roster on every upload satisfies the mirror top-N gate. for i in range(players): healer = i % 2 == 1 conn.execute( "INSERT INTO combatants (encounter_id, name, ally, is_player, cls, encdps, enchps) " - "VALUES (?, ?, 1, 1, ?, ?, ?)", + "VALUES (%s, %s, 1, 1, %s, %s, %s)", (enc_id, f"Player{i}", "Templar" if healer else "Wizard", 1_000_000 - i * 50_000, 300_000 if healer else 0), ) conn.commit() @@ -130,7 +129,7 @@ def test_collect_filters_trash_and_small_groups(seeded_db): def test_collect_other_guild_and_hidden_excluded(seeded_db): _insert_fight(seeded_db, guild="Other Guild") hidden = _insert_fight(seeded_db, started_at=NOW + 500, uploaded_by="RaiderB") - seeded_db.execute("UPDATE encounters SET hidden_at = ? WHERE id = ?", (NOW, hidden)) + seeded_db.execute("UPDATE encounters SET hidden_at = %s WHERE id = %s", (NOW, hidden)) seeded_db.commit() assert collect_new_fights("Varsoon", "Exordium", 0, NOW + 3600) == [] diff --git a/tests/conftest.py b/tests/conftest.py index 2a8f398a..977c07f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,7 +65,6 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 BE-096: moved from module-level os.environ calls to avoid a race with pytest plugins (e.g. pytest-asyncio) that may import web.app during plugin discovery.""" - os.environ["DB_CENSUS_PATH"] = str(_TEST_DB_DIR / "backend.census.db") os.environ["DB_ZONES_PATH"] = str(_TEST_DB_DIR / "zones.db") os.environ["DB_SPELLS_PATH"] = str(_TEST_DB_DIR / "spells.db") os.environ["DB_RECIPES_PATH"] = str(_TEST_DB_DIR / "recipes.db") @@ -103,7 +102,6 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 # Imports below this line read the env vars above when they evaluate their # module-level constants (DB_PATH, SESSION_SECRET, ...). - from backend.census import store as census_store # Force module-level DB_PATH constants to pick up the env vars set above. # The constants are evaluated at module import time; if a pytest plugin @@ -121,8 +119,6 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 from backend.eq2db import spells as spells_db from backend.eq2db import zones as zones_db - census_store.DB_PATH = resolve_db_path("DB_CENSUS_PATH", "census", "census.db") - census_store.store.path = census_store.DB_PATH # eq2db catalogue modules: re-point both the module constant AND the # shared catalogue instance (its path was captured at import time). for mod, env_var, subdir, filename in ( @@ -226,6 +222,7 @@ def mock_character_cache(): # Re-export per-domain fixtures so they can be requested from any test # directory (the fixtures' module location is implementation detail). +from tests.fixtures.census_db import census_schema # noqa: F401,E402 from tests.fixtures.logging_state import _logging_state_isolation # noqa: F401,E402 from tests.fixtures.parses_db import parses_db_conn, parses_db_path # noqa: F401,E402 from tests.fixtures.pg import users_schema # noqa: F401,E402 diff --git a/tests/fixtures/census_db.py b/tests/fixtures/census_db.py new file mode 100644 index 00000000..930f6571 --- /dev/null +++ b/tests/fixtures/census_db.py @@ -0,0 +1,31 @@ +"""Shared census-family fixture (Postgres ``census`` schema). + +``census_schema`` leases an isolated scratch schema built from +db/migrations/0003_census.sql, re-points the shared CensusStore at it, +and yields the schema name — the analog of the old +``CensusStore(tmp_db)`` + ``store.path`` re-point. +""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest + +from backend.census import store as census_store_module +from tests.fixtures.pg import SchemaLeaser + +#: Scratch-schema pool for the census family (census_s_N). +census_leaser = SchemaLeaser("census", "0003_census.sql") + + +@pytest.fixture +def census_schema(monkeypatch: pytest.MonkeyPatch) -> Generator[str]: + """Isolated leased census schema with the shared store re-pointed.""" + name = census_leaser.acquire() + monkeypatch.setattr(census_store_module, "SCHEMA", name) + monkeypatch.setattr(census_store_module.store, "schema", name) + try: + yield name + finally: + census_leaser.release(name) diff --git a/tests/server/parses/test_cleanup.py b/tests/server/parses/test_cleanup.py index 5fca0d56..b4f8b11c 100644 --- a/tests/server/parses/test_cleanup.py +++ b/tests/server/parses/test_cleanup.py @@ -7,7 +7,7 @@ from __future__ import annotations -import sqlite3 +from typing import Any import pytest @@ -20,18 +20,17 @@ @pytest.fixture -def seeded_db(tmp_path, monkeypatch): - """Temp parses DB wired into both the seeding conn and the sweep (via - DB_PATH). Returns the seed connection.""" - db_path = tmp_path / "parses.db" - monkeypatch.setattr(parses_db.store, "path", db_path) - conn = parses_db.ParsesStore(db_path).init_db() +def seeded_db(parses_db_path: str): + """Leased scratch parses schema wired into both the seeding conn and the + sweep (the fixture re-points the shared store). Returns the seed + connection.""" + conn = parses_db.ParsesStore(parses_db_path).init_db() yield conn conn.close() def _insert( - conn: sqlite3.Connection, + conn: Any, *, title: str, started_at: int, @@ -43,10 +42,10 @@ def _insert( guild_name: str | None = "Exordium", zone: str = "Castle Mistmoore", ) -> int: - cur = conn.execute( + row = conn.execute( "INSERT INTO encounters (world, act_encid, title, zone, started_at, ended_at, " "duration_s, success_level, source_dsn, uploaded_by, guild_name, ingested_at, hidden_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'eq2act', ?, ?, ?, ?)", + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'eq2act', %s, %s, %s, %s) RETURNING id", ( world, f"enc-{world}-{started_at}-{uploaded_by}", @@ -61,13 +60,13 @@ def _insert( started_at, hidden_at, ), - ) + ).fetchone() conn.commit() - return int(cur.lastrowid) + return int(row["id"]) -def _ids(conn: sqlite3.Connection) -> set[int]: - return {r[0] for r in conn.execute("SELECT id FROM encounters").fetchall()} +def _ids(conn: Any) -> set[int]: + return {r["id"] for r in conn.execute("SELECT id FROM encounters").fetchall()} def test_trash_deleted_after_cutoff_recent_survives(seeded_db): @@ -184,5 +183,5 @@ def test_nothing_deleted_inside_retention_window(seeded_db): result = parse_cleanup.run_parse_cleanup(now=NOW, retention_days=3) - assert result == {"trash_deleted": 0, "dup_uploads_deleted": 0} + assert result == {"trash_deleted": 0, "dup_uploads_deleted": 0, "detail_pruned": 0} assert len(_ids(seeded_db)) == 3 diff --git a/tests/server/parses/test_db.py b/tests/server/parses/test_db.py index 2509e548..d58aba8b 100644 --- a/tests/server/parses/test_db.py +++ b/tests/server/parses/test_db.py @@ -1,123 +1,76 @@ -"""Tests for parses.db — schema, migrations, helpers.""" +"""Tests for parses.db — schema invariants (migrations-owned DDL) + helpers.""" from __future__ import annotations -import sqlite3 from datetime import datetime +import psycopg.errors import pytest +from backend import pg_migrate from backend.server.parses import db as parses_db from backend.server.parses.models import AttackType, Combatant, DamageType, Encounter +from tests.fixtures.pg import pg_conn -class TestInitDb: - def test_creates_all_tables(self, parses_db_conn): - tables = {r[0] for r in parses_db_conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} +class TestSchema: + """The schema DDL is owned by db/migrations/0002_parses.sql — these pin + the invariants the old SQLite init_db/migration tests guarded.""" + + def test_creates_all_tables(self, parses_db_path): + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema = %s", + (parses_db_path,), + ).fetchall() + tables = {r["table_name"] for r in rows} assert tables >= {"encounters", "combatants", "damage_types", "attack_types", "ingest_log"} - def test_creates_indexes(self, parses_db_conn): - indexes = {r[0] for r in parses_db_conn.execute("SELECT name FROM sqlite_master WHERE type='index'").fetchall()} + def test_creates_indexes(self, parses_db_path): + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT indexname FROM pg_indexes WHERE schemaname = %s", + (parses_db_path,), + ).fetchall() + indexes = {r["indexname"] for r in rows} assert "idx_encounters_started_desc" in indexes assert "idx_attack_types_damage_desc" in indexes assert "idx_combatants_ally" in indexes - def test_migrations_idempotent(self, parses_db_conn): - # Re-running every CREATE / migration on the same connection should be safe. - for stmt in ( - parses_db._CREATE_ENCOUNTERS, - parses_db._CREATE_COMBATANTS, - parses_db._CREATE_DAMAGE_TYPES, - parses_db._CREATE_ATTACK_TYPES, - parses_db._CREATE_INGEST_LOG, - ): - parses_db_conn.execute(stmt) - for idx in parses_db._CREATE_INDEXES: - parses_db_conn.execute(idx) - # Migration runner is also idempotent on an already-migrated DB. - parses_db.store._migrate_attack_types_unique(parses_db_conn) - - def test_encounters_has_hidden_at_column(self, parses_db_conn): - cols = [r[1] for r in parses_db_conn.execute("PRAGMA table_info(encounters)").fetchall()] + def test_migrations_idempotent(self): + """Descendant of the SQLite "re-run every CREATE / ALTER" check: + re-running the migration runner on an already-migrated database is + a no-op (the ledger skips every applied file).""" + assert pg_migrate.run() == [] + + def test_encounters_has_hidden_at_column(self, parses_db_path): + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'encounters'", + (parses_db_path,), + ).fetchall() + cols = {r["column_name"] for r in rows} assert "hidden_at" in cols - def test_migrates_legacy_attack_types_unique(self): - """A DB created with the old UNIQUE(combatant_id, attack_name) - constraint gets transparently recreated with the new tuple, and - existing rows are preserved.""" - conn = sqlite3.connect(":memory:") - try: - conn.execute("PRAGMA foreign_keys = ON;") - # Hand-build the legacy schema (encounters + combatants minimal, - # attack_types with the OLD UNIQUE). - conn.execute(parses_db._CREATE_ENCOUNTERS) - conn.execute(parses_db._CREATE_COMBATANTS) - conn.execute(""" - CREATE TABLE attack_types ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - combatant_id INTEGER NOT NULL, - victim TEXT, - swing_type INTEGER NOT NULL DEFAULT 0, - attack_name TEXT NOT NULL, - started_at INTEGER NOT NULL DEFAULT 0, - ended_at INTEGER NOT NULL DEFAULT 0, - duration_s INTEGER NOT NULL DEFAULT 0, - damage INTEGER NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - char_dps REAL NOT NULL DEFAULT 0, - dps REAL NOT NULL DEFAULT 0, - average REAL NOT NULL DEFAULT 0, - median INTEGER NOT NULL DEFAULT 0, - min_hit INTEGER NOT NULL DEFAULT 0, - max_hit INTEGER NOT NULL DEFAULT 0, - resist TEXT, - hits INTEGER NOT NULL DEFAULT 0, - crit_hits INTEGER NOT NULL DEFAULT 0, - blocked INTEGER NOT NULL DEFAULT 0, - misses INTEGER NOT NULL DEFAULT 0, - swings INTEGER NOT NULL DEFAULT 0, - to_hit REAL NOT NULL DEFAULT 0, - average_delay REAL NOT NULL DEFAULT 0, - crit_perc REAL NOT NULL DEFAULT 0, - crit_types TEXT, - FOREIGN KEY (combatant_id) REFERENCES combatants(id) ON DELETE CASCADE, - UNIQUE (combatant_id, attack_name) - ) - """) - # Seed an encounter + combatant + one attack_types row. - conn.execute( - "INSERT INTO encounters (act_encid, title, zone, started_at, ended_at, " - "duration_s, total_damage, encdps, kills, deaths, source_dsn, ingested_at) " - "VALUES ('legacy', 't', 'z', 0, 0, 0, 0, 0, 0, 0, 'eq2act', 0)" - ) - eid = conn.execute("SELECT id FROM encounters").fetchone()[0] - conn.execute( - "INSERT INTO combatants (encounter_id, name, ally, started_at, ended_at, " - "duration_s, damage, damage_perc, kills, healed, healed_perc, crit_heals, " - "heals, cure_dispels, power_drain, power_replenish, dps, encdps, enchps, " - "hits, crit_hits, blocked, misses, swings, heals_taken, damage_taken, " - "deaths, to_hit, crit_dam_perc, crit_heal_perc, crit_types, threat_str, " - "threat_delta) VALUES (?, 'M', 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, " - "0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', 0)", - (eid,), - ) - cid = conn.execute("SELECT id FROM combatants").fetchone()[0] - conn.execute( - "INSERT INTO attack_types (combatant_id, swing_type, attack_name, damage) VALUES (?, 1, 'Smite', 500)", - (cid,), - ) - # Run the migration. - parses_db.store._migrate_attack_types_unique(conn) - # Existing row survived. - assert conn.execute("SELECT damage FROM attack_types WHERE attack_name = 'Smite'").fetchone()[0] == 500 - # New constraint now allows same-name across swing types. - conn.execute( - "INSERT INTO attack_types (combatant_id, swing_type, attack_name, damage) VALUES (?, 2, 'Smite', 999)", - (cid,), - ) - assert conn.execute("SELECT COUNT(*) FROM attack_types WHERE attack_name = 'Smite'").fetchone()[0] == 2 - finally: - conn.close() + def test_attack_types_unique_tuple_is_swing_scoped(self, parses_db_path): + """Descendant of the SQLite legacy UNIQUE(combatant_id, attack_name) + rebuild migration: on Postgres the (combatant_id, swing_type, + attack_name) tuple is part of the schema outright. The behavioural + halves (same name across swing types allowed / same tuple collides) + are pinned in TestUniqueConstraints.""" + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT kcu.column_name FROM information_schema.table_constraints tc" + " JOIN information_schema.key_column_usage kcu" + " ON kcu.constraint_name = tc.constraint_name" + " AND kcu.constraint_schema = tc.constraint_schema" + " WHERE tc.table_schema = %s AND tc.table_name = 'attack_types'" + " AND tc.constraint_type = 'UNIQUE'" + " ORDER BY kcu.ordinal_position", + (parses_db_path,), + ).fetchall() + assert [r["column_name"] for r in rows] == ["combatant_id", "swing_type", "attack_name"] def _sample_encounter() -> Encounter: @@ -192,10 +145,10 @@ def test_insert_encounter_writes_uploaded_by(self, parses_db_conn): uploaded_by="Menludiir", ) row = parses_db_conn.execute( - "SELECT uploaded_by FROM encounters WHERE act_encid = ?", + "SELECT uploaded_by FROM encounters WHERE act_encid = %s", ("18cf3eb9",), ).fetchone() - assert row[0] == "Menludiir" + assert row["uploaded_by"] == "Menludiir" def test_insert_encounter_defaults_uploaded_by_to_local(self, parses_db_conn): parses_db.store.insert_encounter( @@ -205,10 +158,10 @@ def test_insert_encounter_defaults_uploaded_by_to_local(self, parses_db_conn): ingested_at=1700000000, ) row = parses_db_conn.execute( - "SELECT uploaded_by FROM encounters WHERE act_encid = ?", + "SELECT uploaded_by FROM encounters WHERE act_encid = %s", ("18cf3eb9",), ).fetchone() - assert row[0] == "local" + assert row["uploaded_by"] == "local" def test_insert_encounter_writes_guild_name(self, parses_db_conn): parses_db.store.insert_encounter( @@ -220,10 +173,10 @@ def test_insert_encounter_writes_guild_name(self, parses_db_conn): guild_name="Exordium", ) row = parses_db_conn.execute( - "SELECT guild_name FROM encounters WHERE act_encid = ?", + "SELECT guild_name FROM encounters WHERE act_encid = %s", ("18cf3eb9",), ).fetchone() - assert row[0] == "Exordium" + assert row["guild_name"] == "Exordium" def test_insert_encounter_defaults_guild_to_null(self, parses_db_conn): parses_db.store.insert_encounter( @@ -233,10 +186,10 @@ def test_insert_encounter_defaults_guild_to_null(self, parses_db_conn): ingested_at=1700000000, ) row = parses_db_conn.execute( - "SELECT guild_name FROM encounters WHERE act_encid = ?", + "SELECT guild_name FROM encounters WHERE act_encid = %s", ("18cf3eb9",), ).fetchone() - assert row[0] is None + assert row["guild_name"] is None def test_insert_combatants_writes_snapshot(self, parses_db_conn): from backend.server.parses.models import CombatantSnapshot @@ -296,24 +249,24 @@ def test_soft_delete_sets_hidden_at_and_by(self, parses_db_conn): parses_db.store.soft_delete_encounter(parses_db_conn, eid, hidden_at=1700001111, hidden_by="admin-1") is True ) - row = parses_db_conn.execute("SELECT hidden_at, hidden_by FROM encounters WHERE id = ?", (eid,)).fetchone() - assert tuple(row) == (1700001111, "admin-1") + row = parses_db_conn.execute("SELECT hidden_at, hidden_by FROM encounters WHERE id = %s", (eid,)).fetchone() + assert (row["hidden_at"], row["hidden_by"]) == (1700001111, "admin-1") # Idempotent: re-soft-deleting an already-hidden row is a no-op (returns False) # and never overwrites the original actor. assert ( parses_db.store.soft_delete_encounter(parses_db_conn, eid, hidden_at=1700002222, hidden_by="admin-2") is False ) - row = parses_db_conn.execute("SELECT hidden_by FROM encounters WHERE id = ?", (eid,)).fetchone() - assert row[0] == "admin-1" + row = parses_db_conn.execute("SELECT hidden_by FROM encounters WHERE id = %s", (eid,)).fetchone() + assert row["hidden_by"] == "admin-1" def test_unhide_encounter_clears_marker(self, parses_db_conn): enc = _sample_encounter() eid = parses_db.store.insert_encounter(parses_db_conn, enc, source_dsn="eq2act", ingested_at=1700000000) parses_db.store.soft_delete_encounter(parses_db_conn, eid, hidden_at=1700001111, hidden_by="admin-1") assert parses_db.store.unhide_encounter(parses_db_conn, eid) is True - row = parses_db_conn.execute("SELECT hidden_at, hidden_by FROM encounters WHERE id = ?", (eid,)).fetchone() - assert tuple(row) == (None, None) + row = parses_db_conn.execute("SELECT hidden_at, hidden_by FROM encounters WHERE id = %s", (eid,)).fetchone() + assert (row["hidden_at"], row["hidden_by"]) == (None, None) # Already-visible row → no-op, returns False. assert parses_db.store.unhide_encounter(parses_db_conn, eid) is False @@ -461,7 +414,7 @@ def test_duplicate_act_encid_rejected(self, parses_db_conn): source_dsn="eq2act", ingested_at=1700000000, ) - with pytest.raises(sqlite3.IntegrityError): + with pytest.raises(psycopg.errors.UniqueViolation): parses_db.store.insert_encounter( parses_db_conn, _sample_encounter(), @@ -479,7 +432,7 @@ def test_duplicate_combatant_in_encounter_rejected(self, parses_db_conn): ) cs = [_sample_combatant("Menludiir", ally=True, damage=1)] parses_db.store.insert_combatants_bulk(parses_db_conn, eid, cs) - with pytest.raises(sqlite3.IntegrityError): + with pytest.raises(psycopg.errors.UniqueViolation): parses_db.store.insert_combatants_bulk(parses_db_conn, eid, cs) def test_same_attack_name_across_swing_types_allowed(self, parses_db_conn): @@ -496,21 +449,21 @@ def test_same_attack_name_across_swing_types_allowed(self, parses_db_conn): parses_db_conn, eid, [_sample_combatant("Menludiir", ally=True, damage=1)] ) cid = parses_db_conn.execute( - "SELECT id FROM combatants WHERE encounter_id = ? AND name = ?", + "SELECT id FROM combatants WHERE encounter_id = %s AND name = %s", (eid, "Menludiir"), - ).fetchone()[0] + ).fetchone()["id"] parses_db_conn.executemany( "INSERT INTO attack_types (combatant_id, victim, swing_type, attack_name, " "damage, hits, swings, crit_hits, max_hit, resist) " - "VALUES (?, '', ?, 'Cleanse', ?, 1, 1, 0, 0, '')", + "VALUES (%s, '', %s, 'Cleanse', %s, 1, 1, 0, 0, '')", [(cid, 2, 1000), (cid, 3, 500)], ) rows = parses_db_conn.execute( "SELECT swing_type, damage FROM attack_types " - "WHERE combatant_id = ? AND attack_name = 'Cleanse' ORDER BY swing_type", + "WHERE combatant_id = %s AND attack_name = 'Cleanse' ORDER BY swing_type", (cid,), ).fetchall() - assert [(r[0], r[1]) for r in rows] == [(2, 1000), (3, 500)] + assert [(r["swing_type"], r["damage"]) for r in rows] == [(2, 1000), (3, 500)] def test_duplicate_attack_within_same_swing_type_rejected(self, parses_db_conn): """The new tuple is (combatant_id, swing_type, attack_name) — same @@ -525,20 +478,20 @@ def test_duplicate_attack_within_same_swing_type_rejected(self, parses_db_conn): parses_db_conn, eid, [_sample_combatant("Menludiir", ally=True, damage=1)] ) cid = parses_db_conn.execute( - "SELECT id FROM combatants WHERE encounter_id = ? AND name = ?", + "SELECT id FROM combatants WHERE encounter_id = %s AND name = %s", (eid, "Menludiir"), - ).fetchone()[0] + ).fetchone()["id"] parses_db_conn.execute( "INSERT INTO attack_types (combatant_id, victim, swing_type, attack_name, " "damage, hits, swings, crit_hits, max_hit, resist) " - "VALUES (?, '', 2, 'Cleanse', 1000, 1, 1, 0, 0, '')", + "VALUES (%s, '', 2, 'Cleanse', 1000, 1, 1, 0, 0, '')", (cid,), ) - with pytest.raises(sqlite3.IntegrityError): + with pytest.raises(psycopg.errors.UniqueViolation): parses_db_conn.execute( "INSERT INTO attack_types (combatant_id, victim, swing_type, attack_name, " "damage, hits, swings, crit_hits, max_hit, resist) " - "VALUES (?, '', 2, 'Cleanse', 2000, 1, 1, 0, 0, '')", + "VALUES (%s, '', 2, 'Cleanse', 2000, 1, 1, 0, 0, '')", (cid,), ) @@ -622,7 +575,7 @@ def _seed(self, parses_db_conn): INSERT INTO attack_types ( combatant_id, victim, swing_type, attack_name, damage, hits, swings, crit_hits, max_hit, resist - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, [ (cid, "", 1, "crush", 7000, 10, 10, 1, 1500, "crushing"), @@ -668,7 +621,7 @@ def test_top_threats_excludes_All_rollup(self, parses_db_conn): parses_db_conn.execute( "INSERT INTO attack_types (combatant_id, victim, swing_type, attack_name, " "damage, hits, swings, crit_hits, max_hit, resist) " - "VALUES (?, '', 100, 'All', 999999, 999, 999, 0, 0, 'All')", + "VALUES (%s, '', 100, 'All', 999999, 999, 999, 0, 0, 'All')", (cid,), ) threats = parses_db.store.get_top_threats_for_combatant(parses_db_conn, cid) @@ -716,7 +669,7 @@ def _seed(self, conn, *, encid: str, guild_name: str, uploaded_by: str = "Menlud def test_delete_encounter_removes_row(self, parses_db_conn): eid = self._seed(parses_db_conn, encid="enc1", guild_name="Exordium") assert parses_db.store.delete_encounter(parses_db_conn, eid) is True - assert parses_db_conn.execute("SELECT COUNT(*) FROM encounters").fetchone()[0] == 0 + assert parses_db_conn.execute("SELECT COUNT(*) AS n FROM encounters").fetchone()["n"] == 0 def test_delete_encounter_returns_false_when_missing(self, parses_db_conn): assert parses_db.store.delete_encounter(parses_db_conn, 99999) is False @@ -725,18 +678,30 @@ def test_delete_encounter_cascades_children(self, parses_db_conn): eid = self._seed(parses_db_conn, encid="enc2", guild_name="Exordium") # Sanity: rows exist before delete. assert ( - parses_db_conn.execute("SELECT COUNT(*) FROM combatants WHERE encounter_id = ?", (eid,)).fetchone()[0] > 0 + parses_db_conn.execute("SELECT COUNT(*) AS n FROM combatants WHERE encounter_id = %s", (eid,)).fetchone()[ + "n" + ] + > 0 ) assert ( - parses_db_conn.execute("SELECT COUNT(*) FROM ingest_log WHERE encounter_id = ?", (eid,)).fetchone()[0] == 1 + parses_db_conn.execute("SELECT COUNT(*) AS n FROM ingest_log WHERE encounter_id = %s", (eid,)).fetchone()[ + "n" + ] + == 1 ) parses_db.store.delete_encounter(parses_db_conn, eid) # All children gone via FK cascade. assert ( - parses_db_conn.execute("SELECT COUNT(*) FROM combatants WHERE encounter_id = ?", (eid,)).fetchone()[0] == 0 + parses_db_conn.execute("SELECT COUNT(*) AS n FROM combatants WHERE encounter_id = %s", (eid,)).fetchone()[ + "n" + ] + == 0 ) assert ( - parses_db_conn.execute("SELECT COUNT(*) FROM ingest_log WHERE encounter_id = ?", (eid,)).fetchone()[0] == 0 + parses_db_conn.execute("SELECT COUNT(*) AS n FROM ingest_log WHERE encounter_id = %s", (eid,)).fetchone()[ + "n" + ] + == 0 ) @@ -749,13 +714,23 @@ class TestWorldScoping: """Verify that world is stored, uniqueness is per (world, act_encid), and all lookup helpers honour the world filter.""" - def test_encounters_has_world_column(self, parses_db_conn): - cols = [r[1] for r in parses_db_conn.execute("PRAGMA table_info(encounters)").fetchall()] - assert "world" in cols - - def test_ingest_log_has_world_column(self, parses_db_conn): - cols = [r[1] for r in parses_db_conn.execute("PRAGMA table_info(ingest_log)").fetchall()] - assert "world" in cols + def test_encounters_has_world_column(self, parses_db_path): + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'encounters'", + (parses_db_path,), + ).fetchall() + assert "world" in {r["column_name"] for r in rows} + + def test_ingest_log_has_world_column(self, parses_db_path): + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'ingest_log'", + (parses_db_path,), + ).fetchall() + assert "world" in {r["column_name"] for r in rows} def test_world_stored_on_encounter(self, parses_db_conn): parses_db.store.insert_encounter( @@ -765,8 +740,8 @@ def test_world_stored_on_encounter(self, parses_db_conn): ingested_at=1700000000, world="Wuoshi", ) - row = parses_db_conn.execute("SELECT world FROM encounters WHERE act_encid = ?", ("18cf3eb9",)).fetchone() - assert row[0] == "Wuoshi" + row = parses_db_conn.execute("SELECT world FROM encounters WHERE act_encid = %s", ("18cf3eb9",)).fetchone() + assert row["world"] == "Wuoshi" def test_same_act_encid_different_world_both_insert(self, parses_db_conn): """The UNIQUE constraint is (world, act_encid), so the same encid @@ -786,14 +761,14 @@ def test_same_act_encid_different_world_both_insert(self, parses_db_conn): world="Wuoshi", ) assert eid_v != eid_w - count = parses_db_conn.execute("SELECT COUNT(*) FROM encounters WHERE act_encid = ?", ("18cf3eb9",)).fetchone()[ - 0 - ] + count = parses_db_conn.execute( + "SELECT COUNT(*) AS n FROM encounters WHERE act_encid = %s", ("18cf3eb9",) + ).fetchone()["n"] assert count == 2 def test_same_world_same_encid_still_collides(self, parses_db_conn): """Duplicate (world, act_encid) within the same server must still - raise an IntegrityError.""" + raise a UniqueViolation.""" parses_db.store.insert_encounter( parses_db_conn, _sample_encounter(), @@ -801,7 +776,7 @@ def test_same_world_same_encid_still_collides(self, parses_db_conn): ingested_at=1700000000, world="Varsoon", ) - with pytest.raises(sqlite3.IntegrityError): + with pytest.raises(psycopg.errors.UniqueViolation): parses_db.store.insert_encounter( parses_db_conn, _sample_encounter(), @@ -844,8 +819,8 @@ def test_ingest_log_world_stored(self, parses_db_conn): ingested_at=1700000000, world="Kaladim", ) - row = parses_db_conn.execute("SELECT world FROM ingest_log WHERE act_encid = ?", (enc.encid,)).fetchone() - assert row[0] == "Kaladim" + row = parses_db_conn.execute("SELECT world FROM ingest_log WHERE act_encid = %s", (enc.encid,)).fetchone() + assert row["world"] == "Kaladim" def test_find_encounter_by_act_encid_world_scoped(self, parses_db_conn): """find_encounter_by_act_encid must only return the row for the @@ -902,140 +877,28 @@ def test_encounters_default_world_is_varsoon(self, parses_db_conn): source_dsn="eq2act", ingested_at=1700000000, ) - row = parses_db_conn.execute("SELECT world FROM encounters WHERE act_encid = ?", ("18cf3eb9",)).fetchone() - assert row[0] == "Varsoon" - - def test_legacy_db_rebuild_preserves_rows(self): - """Simulate an existing DB without a world column: after init_db, - existing encounters must be migrated with world='Varsoon', all ids - preserved, and combatant FK rows intact.""" - conn = sqlite3.connect(":memory:") - try: - conn.execute("PRAGMA foreign_keys = ON;") - # Build the OLD schema (act_encid UNIQUE, no world column). - conn.execute(""" - CREATE TABLE encounters ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - act_encid TEXT NOT NULL UNIQUE, - title TEXT NOT NULL, - zone TEXT, - started_at INTEGER NOT NULL, - ended_at INTEGER NOT NULL, - duration_s INTEGER NOT NULL, - total_damage INTEGER NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - kills INTEGER NOT NULL DEFAULT 0, - deaths INTEGER NOT NULL DEFAULT 0, - success_level INTEGER NOT NULL DEFAULT 0, - source_dsn TEXT NOT NULL, - uploaded_by TEXT NOT NULL DEFAULT 'local', - guild_name TEXT, - ingested_at INTEGER NOT NULL, - hidden_at INTEGER - ) - """) - conn.execute(""" - CREATE TABLE combatants ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - encounter_id INTEGER NOT NULL, - name TEXT NOT NULL, - ally INTEGER NOT NULL DEFAULT 0, - started_at INTEGER NOT NULL DEFAULT 0, - ended_at INTEGER NOT NULL DEFAULT 0, - duration_s INTEGER NOT NULL DEFAULT 0, - damage INTEGER NOT NULL DEFAULT 0, - damage_perc REAL NOT NULL DEFAULT 0, - kills INTEGER NOT NULL DEFAULT 0, - healed INTEGER NOT NULL DEFAULT 0, - healed_perc REAL NOT NULL DEFAULT 0, - crit_heals INTEGER NOT NULL DEFAULT 0, - heals INTEGER NOT NULL DEFAULT 0, - cure_dispels INTEGER NOT NULL DEFAULT 0, - power_drain INTEGER NOT NULL DEFAULT 0, - power_replenish INTEGER NOT NULL DEFAULT 0, - dps REAL NOT NULL DEFAULT 0, - encdps REAL NOT NULL DEFAULT 0, - enchps REAL NOT NULL DEFAULT 0, - hits INTEGER NOT NULL DEFAULT 0, - crit_hits INTEGER NOT NULL DEFAULT 0, - blocked INTEGER NOT NULL DEFAULT 0, - misses INTEGER NOT NULL DEFAULT 0, - swings INTEGER NOT NULL DEFAULT 0, - heals_taken INTEGER NOT NULL DEFAULT 0, - damage_taken INTEGER NOT NULL DEFAULT 0, - deaths INTEGER NOT NULL DEFAULT 0, - to_hit REAL NOT NULL DEFAULT 0, - crit_dam_perc REAL NOT NULL DEFAULT 0, - crit_heal_perc REAL NOT NULL DEFAULT 0, - crit_types TEXT, - threat_str TEXT, - threat_delta INTEGER NOT NULL DEFAULT 0, - level INTEGER, - guild_name TEXT, - cls TEXT, - ilvl REAL, - FOREIGN KEY (encounter_id) REFERENCES encounters(id) ON DELETE CASCADE, - UNIQUE (encounter_id, name) - ) - """) - conn.execute(""" - CREATE TABLE ingest_log ( - act_encid TEXT PRIMARY KEY, - encounter_id INTEGER NOT NULL, - ingested_at INTEGER NOT NULL, - source_dsn TEXT NOT NULL, - FOREIGN KEY (encounter_id) REFERENCES encounters(id) ON DELETE CASCADE - ) - """) - # Seed old-schema rows. - conn.execute( - "INSERT INTO encounters (id, act_encid, title, zone, started_at, ended_at, " - "duration_s, total_damage, encdps, kills, deaths, source_dsn, ingested_at) " - "VALUES (42, 'legacy01', 'OldBoss', 'OldZone', 0, 0, 30, 1000, 33.3, 1, 0, 'eq2act', 0)" - ) - conn.execute( - "INSERT INTO combatants (encounter_id, name, ally, started_at, ended_at, " - "duration_s, damage, damage_perc, kills, healed, healed_perc, crit_heals, " - "heals, cure_dispels, power_drain, power_replenish, dps, encdps, enchps, " - "hits, crit_hits, blocked, misses, swings, heals_taken, damage_taken, " - "deaths, to_hit, crit_dam_perc, crit_heal_perc) VALUES " - "(42, 'OldPlayer', 1, 0, 0, 30, 1000, 100.0, 1, 0, 0.0, 0, 0, 0, 0, 0, " - "33.3, 33.3, 0.0, 10, 5, 0, 0, 10, 0, 100, 0, 100.0, 50.0, 0.0)" - ) - conn.execute( - "INSERT INTO ingest_log (act_encid, encounter_id, ingested_at, source_dsn) " - "VALUES ('legacy01', 42, 0, 'eq2act')" - ) - conn.commit() - # Run the migration (same logic init_db calls). - parses_db.store._migrate_encounters_add_world(conn) - parses_db.store._migrate_ingest_log_add_world(conn) - conn.commit() - # Encounter id preserved, world backfilled. - row = conn.execute("SELECT id, world FROM encounters WHERE act_encid = 'legacy01'").fetchone() - assert row[0] == 42, "id must be preserved after rebuild" - assert row[1] == "Varsoon", "world must be backfilled to Varsoon" - # Combatant FK still resolves. - comb_enc_id = conn.execute("SELECT encounter_id FROM combatants WHERE name = 'OldPlayer'").fetchone()[0] - assert comb_enc_id == 42, "combatant FK must still point at id=42" - # ingest_log world backfilled. - log_world = conn.execute("SELECT world FROM ingest_log WHERE act_encid = 'legacy01'").fetchone()[0] - assert log_world == "Varsoon" - # No dangling FK references — the rename must NOT have rewritten - # combatants' FK clause to point at encounters_old (which no longer - # exists). PRAGMA foreign_key_check returns a row for every violation; - # an empty result means the schema is clean. - fk_violations = list(conn.execute("PRAGMA foreign_key_check")) - assert fk_violations == [], f"dangling FK references after migration: {fk_violations}" - # ON DELETE CASCADE must still fire: deleting the encounter must - # remove its child combatant row. - conn.execute("PRAGMA foreign_keys = ON;") - conn.execute("DELETE FROM encounters WHERE id = 42") - conn.commit() - child_count = conn.execute("SELECT COUNT(*) FROM combatants WHERE encounter_id = 42").fetchone()[0] - assert child_count == 0, "cascade delete must remove child combatants" - # Migrations are idempotent (re-running is a no-op). - parses_db.store._migrate_encounters_add_world(conn) - parses_db.store._migrate_ingest_log_add_world(conn) - finally: - conn.close() + row = parses_db_conn.execute("SELECT world FROM encounters WHERE act_encid = %s", ("18cf3eb9",)).fetchone() + assert row["world"] == "Varsoon" + + def test_world_defaults_to_varsoon_at_the_schema_level(self, parses_db_conn): + """Descendant of the SQLite add-world rebuild migration (which + backfilled legacy rows to 'Varsoon'): on Postgres the DEFAULT lives + in the migrations-owned DDL outright — a raw INSERT that omits the + world column must land as 'Varsoon' in both encounters and + ingest_log. (The FK-cascade half of the old legacy-rebuild test is + pinned in TestDeleteHelpers.test_delete_encounter_cascades_children.) + """ + row = parses_db_conn.execute( + "INSERT INTO encounters (act_encid, title, zone, started_at, ended_at, " + "duration_s, total_damage, encdps, kills, deaths, source_dsn, ingested_at) " + "VALUES ('legacy01', 'OldBoss', 'OldZone', 0, 0, 30, 1000, 33.3, 1, 0, 'eq2act', 0) " + "RETURNING id, world" + ).fetchone() + assert row["world"] == "Varsoon" + parses_db_conn.execute( + "INSERT INTO ingest_log (act_encid, encounter_id, ingested_at, source_dsn) " + "VALUES ('legacy01', %s, 0, 'eq2act')", + (row["id"],), + ) + log = parses_db_conn.execute("SELECT world FROM ingest_log WHERE act_encid = 'legacy01'").fetchone() + assert log["world"] == "Varsoon" diff --git a/tests/server/parses/test_db_is_player.py b/tests/server/parses/test_db_is_player.py index e6312bc9..b474e81e 100644 --- a/tests/server/parses/test_db_is_player.py +++ b/tests/server/parses/test_db_is_player.py @@ -1,100 +1,77 @@ -"""Tests for the Phase-2 is_player schema migration + helpers in parses/db.py. +"""Tests for the Phase-2 is_player column + helpers in parses/db.py. -The migration is idempotent (re-running init_db on a fresh or migrated DB -must not error). The helpers round-trip the classifier's output through -the column. +On Postgres the column and its index are part of the migrations-owned DDL +(db/migrations/0002_parses.sql) rather than an ALTER-style migration; the +idempotency guarantee survives as "re-running the migration runner is a +no-op". The helpers round-trip the classifier's output through the column. """ from __future__ import annotations -import sqlite3 -from pathlib import Path +from typing import Any import pytest +from backend import pg_migrate from backend.server.parses import db as parses_db +from tests.fixtures.pg import pg_conn @pytest.fixture -def conn(tmp_path: Path) -> sqlite3.Connection: - """Fresh throwaway parses DB with the full schema applied.""" - c = parses_db.ParsesStore(tmp_path / "parses.db").init_db() - try: - yield c - finally: - c.close() +def conn(parses_db_conn: Any) -> Any: + """Historical name for the throwaway schema-scoped parses connection.""" + return parses_db_conn -def _insert_encounter(conn: sqlite3.Connection) -> int: - cur = conn.execute( +def _insert_encounter(conn: Any) -> int: + row = conn.execute( """ INSERT INTO encounters ( act_encid, title, zone, started_at, ended_at, duration_s, total_damage, encdps, kills, deaths, source_dsn, ingested_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id """, ("abc123", "Test", "Zone", 1000, 1100, 100, 50000, 500.0, 1, 0, "test", 1100), - ) - return int(cur.lastrowid or 0) + ).fetchone() + return int(row["id"]) if row else 0 -def _insert_combatant(conn: sqlite3.Connection, encounter_id: int, name: str) -> int: - cur = conn.execute( - "INSERT INTO combatants (encounter_id, name, ally) VALUES (?, ?, ?)", +def _insert_combatant(conn: Any, encounter_id: int, name: str) -> int: + row = conn.execute( + "INSERT INTO combatants (encounter_id, name, ally) VALUES (%s, %s, %s) RETURNING id", (encounter_id, name, 1), - ) - return int(cur.lastrowid or 0) - - -def test_is_player_column_exists(tmp_path: Path): - conn = parses_db.ParsesStore(tmp_path / "parses.db").init_db() - try: - cols = {r[1]: r for r in conn.execute("PRAGMA table_info(combatants)")} - assert "is_player" in cols - # DEFAULT NULL allows the lazy-backfill sentinel. - # PRAGMA table_info returns 'NULL' (string) for DEFAULT NULL added via - # ALTER TABLE, and Python None for columns with no default clause — - # either indicates the column is nullable with no non-null default. - assert cols["is_player"][4] in (None, "NULL") - finally: - conn.close() - - -def test_is_player_index_exists(tmp_path: Path): - conn = parses_db.ParsesStore(tmp_path / "parses.db").init_db() - try: - names = { - r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='combatants'") - } - assert "idx_combatants_encounter_is_player" in names - finally: - conn.close() + ).fetchone() + return int(row["id"]) if row else 0 + + +def test_is_player_column_exists(parses_db_path): + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT column_name, is_nullable, column_default FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'combatants'", + (parses_db_path,), + ).fetchall() + cols = {r["column_name"]: r for r in rows} + assert "is_player" in cols + # Nullable with no non-null default — the lazy-backfill sentinel. + assert cols["is_player"]["is_nullable"] == "YES" + assert cols["is_player"]["column_default"] is None + + +def test_is_player_index_exists(parses_db_path): + with pg_conn(parses_db_path) as conn: + rows = conn.execute( + "SELECT indexname FROM pg_indexes WHERE schemaname = %s AND tablename = 'combatants'", + (parses_db_path,), + ).fetchall() + assert "idx_combatants_encounter_is_player" in {r["indexname"] for r in rows} def test_migration_is_idempotent(): - # Re-applying every migration on a fresh DB must not error (the - # OperationalError swallow inside init_db handles duplicate-ALTER). - conn = sqlite3.connect(":memory:") - try: - conn.execute(parses_db._CREATE_ENCOUNTERS) - conn.execute(parses_db._CREATE_COMBATANTS) - # First application — some may already be present in CREATE TABLE - # (e.g. is_player is added by ALTER, not CREATE). - for stmt in parses_db._MIGRATIONS: - try: - conn.execute(stmt) - except sqlite3.OperationalError: - pass - # Second application — every ALTER should now be a no-op. - for stmt in parses_db._MIGRATIONS: - try: - conn.execute(stmt) - except sqlite3.OperationalError: - pass # idempotency: duplicate ALTER must be a no-op - cols = {r[1] for r in conn.execute("PRAGMA table_info(combatants)")} - assert "is_player" in cols - finally: - conn.close() + """The SQLite "re-apply every ALTER, duplicates are no-ops" check becomes: + the migration runner on an already-migrated database applies nothing.""" + assert pg_migrate.run() == [] def test_update_combatant_is_player_round_trip(conn): @@ -103,7 +80,7 @@ def test_update_combatant_is_player_round_trip(conn): b = _insert_combatant(conn, enc_id, "Bravo") c = _insert_combatant(conn, enc_id, "Charlie") parses_db.store.update_combatant_is_player(conn, {a: True, b: False, c: True}) - rows = {r[0]: r[1] for r in conn.execute("SELECT id, is_player FROM combatants ORDER BY id")} + rows = {r["id"]: r["is_player"] for r in conn.execute("SELECT id, is_player FROM combatants ORDER BY id")} assert rows[a] == 1 assert rows[b] == 0 assert rows[c] == 1 @@ -114,8 +91,8 @@ def test_update_combatant_is_player_overwrites_existing(conn): a = _insert_combatant(conn, enc_id, "Alpha") parses_db.store.update_combatant_is_player(conn, {a: True}) parses_db.store.update_combatant_is_player(conn, {a: False}) - row = conn.execute("SELECT is_player FROM combatants WHERE id = ?", (a,)).fetchone() - assert row[0] == 0 + row = conn.execute("SELECT is_player FROM combatants WHERE id = %s", (a,)).fetchone() + assert row["is_player"] == 0 def test_update_combatant_is_player_empty_dict_is_noop(conn): @@ -129,4 +106,4 @@ def test_invalidate_is_player_cache_with_conn_sets_every_row_to_null(conn): parses_db.store.update_combatant_is_player(conn, {a: True, b: False}) parses_db.store.invalidate_is_player_cache_with_conn(conn) rows = conn.execute("SELECT is_player FROM combatants").fetchall() - assert all(r[0] is None for r in rows), rows + assert all(r["is_player"] is None for r in rows), rows diff --git a/tests/server/parses/test_detail_retention.py b/tests/server/parses/test_detail_retention.py new file mode 100644 index 00000000..1c3dc790 --- /dev/null +++ b/tests/server/parses/test_detail_retention.py @@ -0,0 +1,114 @@ +"""Tiered breakdown-detail retention (user decision 2026-10-04). + +Curated raid zones keep attack_types/damage_types 30 days, curated +group instances 14, anything else 7; encounters + combatants live +forever and detail_pruned_at is stamped for the detail-page notice. +The zone→tier classifier is monkeypatched so these tests don't depend +on zones.db content. +""" + +from __future__ import annotations + +import pytest + +from backend.server.parses import cleanup +from backend.server.parses.db import store as parses_db + +NOW = 1_760_000_000 +DAY = 86_400 + +_TIERS = {"Raid Zone": 30, "Group Zone": 14} + +#: Captured before the autouse fixture monkeypatches the module attribute, +#: so the mapping test can exercise the real function. +_REAL_TIER_FN = cleanup.detail_retention_days + + +@pytest.fixture(autouse=True) +def _fake_tiers(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(cleanup, "detail_retention_days", lambda zone: _TIERS.get(zone or "", 7)) + + +def _seed(conn, *, zone: str, age_days: int, title: str = "Boss") -> int: + started = NOW - age_days * DAY + row = conn.execute( + "INSERT INTO encounters (world, act_encid, title, zone, started_at, ended_at, duration_s," + " success_level, source_dsn, uploaded_by, ingested_at)" + " VALUES ('Wuoshi', %s, %s, %s, %s, %s, 300, 1, 'seed', 'u1', %s) RETURNING id", + (f"e-{zone}-{age_days}", title, zone, started, started + 300, NOW), + ).fetchone() + eid = row["id"] + cid = conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, damage, is_player)" + " VALUES (%s, 'Menludiir', 1, 100, 1) RETURNING id", + (eid,), + ).fetchone()["id"] + conn.execute("INSERT INTO attack_types (combatant_id, swing_type, attack_name) VALUES (%s, 2, 'Smite')", (cid,)) + conn.execute("INSERT INTO damage_types (combatant_id, damage_type) VALUES (%s, 'divine')", (cid,)) + conn.commit() + return eid + + +def _detail_counts(conn, eid: int) -> tuple[int, int]: + at = conn.execute( + "SELECT COUNT(*) AS n FROM attack_types a JOIN combatants c ON c.id = a.combatant_id WHERE c.encounter_id = %s", + (eid,), + ).fetchone()["n"] + dt = conn.execute( + "SELECT COUNT(*) AS n FROM damage_types d JOIN combatants c ON c.id = d.combatant_id WHERE c.encounter_id = %s", + (eid,), + ).fetchone()["n"] + return at, dt + + +def test_tiers_prune_only_overdue_zones(parses_db_conn): + conn = parses_db_conn + raid_young = _seed(conn, zone="Raid Zone", age_days=10) + raid_old = _seed(conn, zone="Raid Zone", age_days=31) + dungeon_young = _seed(conn, zone="Group Zone", age_days=10) + dungeon_old = _seed(conn, zone="Group Zone", age_days=15) + other_young = _seed(conn, zone="Backyard", age_days=6) + other_old = _seed(conn, zone="Backyard", age_days=8) + + pruned = cleanup._sweep_detail_retention(conn, NOW) + assert pruned == 3 + + stamps = {r["id"]: r["detail_pruned_at"] for r in conn.execute("SELECT id, detail_pruned_at FROM encounters")} + for eid in (raid_old, dungeon_old, other_old): + assert stamps[eid] == NOW, (eid, stamps) + assert _detail_counts(conn, eid) == (0, 0) + for eid in (raid_young, dungeon_young, other_young): + assert stamps[eid] is None, (eid, stamps) + assert _detail_counts(conn, eid) == (1, 1) + + # Summary rows are untouched. + assert conn.execute("SELECT COUNT(*) AS n FROM encounters").fetchone()["n"] == 6 + assert conn.execute("SELECT COUNT(*) AS n FROM combatants").fetchone()["n"] == 6 + + +def test_sweep_is_idempotent(parses_db_conn): + conn = parses_db_conn + _seed(conn, zone="Backyard", age_days=9) + assert cleanup._sweep_detail_retention(conn, NOW) == 1 + assert cleanup._sweep_detail_retention(conn, NOW) == 0 # already stamped + + +def test_run_parse_cleanup_reports_detail_pruned(parses_db_path, monkeypatch): + # Boss title (capitalised) so the row-retention half keeps the fight. + conn = parses_db.init_db() + try: + _seed(conn, zone="Backyard", age_days=9, title="Venekor") + finally: + conn.close() + out = cleanup.run_parse_cleanup(now=NOW) + assert out["detail_pruned"] == 1, out + assert out["trash_deleted"] == 0 and out["dup_uploads_deleted"] == 0, out + + +def test_default_tier_mapping_uses_zone_classifier(monkeypatch): + import backend.server.api.parses.list as parses_list + + monkeypatch.setattr(parses_list, "_classify_zone", lambda zone: {"R": "raid", "D": "dungeon"}.get(zone, "other")) + assert _REAL_TIER_FN("R") == cleanup.DETAIL_RETENTION_RAID_DAYS + assert _REAL_TIER_FN("D") == cleanup.DETAIL_RETENTION_DUNGEON_DAYS + assert _REAL_TIER_FN("X") == cleanup.DETAIL_RETENTION_OTHER_DAYS diff --git a/tests/server/test_admin_tamper_reports.py b/tests/server/test_admin_tamper_reports.py index 29034000..b8e7218f 100644 --- a/tests/server/test_admin_tamper_reports.py +++ b/tests/server/test_admin_tamper_reports.py @@ -33,13 +33,21 @@ def _fake_admin(request=None): # noqa: ARG001 return _fake_admin_user +@pytest.fixture(autouse=True) +def _parses_schema(parses_db_path: str) -> str: + """Isolated leased parses schema per test — the fixture re-points the + shared store, so the admin route's argless ``store.init_db()`` and the + helpers below hit the same scratch schema.""" + return parses_db_path + + def _seed_tamper_reports(now: int = 1700000000) -> list[int]: """Drop a fixed set of reports into the test DB and return their ids, newest-first so tests can refer to them by index.""" _wipe_tamper_reports() - # The shared store's `path` is re-pointed by conftest, and init_db() - # reads it at call time — so this argless call and the admin route's - # argless call hit the same (redirected) DB. + # The shared store's schema is re-pointed by the autouse fixture above, + # and init_db() reads it at call time — so this argless call and the + # admin route's argless call hit the same leased scratch schema. conn = parses_db.store.init_db() ids: list[int] = [] try: @@ -222,14 +230,14 @@ async def test_acknowledge_flips_pending_row(app): conn = parses_db.store.init_db() try: row = conn.execute( - "SELECT acknowledged_at, acknowledged_by FROM tamper_reports WHERE id = ?", + "SELECT acknowledged_at, acknowledged_by FROM tamper_reports WHERE id = %s", (target,), ).fetchone() finally: conn.close() - assert row[0] is not None # acknowledged_at populated - assert row[1] == "admin-1" # actor + assert row["acknowledged_at"] is not None # acknowledged_at populated + assert row["acknowledged_by"] == "admin-1" # actor @pytest.mark.asyncio @@ -251,12 +259,12 @@ async def test_acknowledge_idempotent_on_already_ack(app): conn = parses_db.store.init_db() try: row = conn.execute( - "SELECT acknowledged_by FROM tamper_reports WHERE id = ?", + "SELECT acknowledged_by FROM tamper_reports WHERE id = %s", (target,), ).fetchone() finally: conn.close() - assert row[0] == "someone-else" + assert row["acknowledged_by"] == "someone-else" @pytest.mark.asyncio @@ -297,8 +305,10 @@ async def test_acknowledge_all_flips_every_pending(app): conn = parses_db.store.init_db() try: - pending = conn.execute("SELECT COUNT(*) FROM tamper_reports WHERE acknowledged_at IS NULL").fetchone()[0] - actor = conn.execute("SELECT acknowledged_by FROM tamper_reports WHERE id = ?", (ids[0],)).fetchone()[0] + pending = conn.execute("SELECT COUNT(*) AS n FROM tamper_reports WHERE acknowledged_at IS NULL").fetchone()["n"] + actor = conn.execute("SELECT acknowledged_by FROM tamper_reports WHERE id = %s", (ids[0],)).fetchone()[ + "acknowledged_by" + ] finally: conn.close() assert pending == 0 # working set is clear @@ -327,7 +337,7 @@ async def test_purge_deletes_only_acknowledged(app): conn = parses_db.store.init_db() try: - remaining = {row[0] for row in conn.execute("SELECT id FROM tamper_reports").fetchall()} + remaining = {row["id"] for row in conn.execute("SELECT id FROM tamper_reports").fetchall()} finally: conn.close() assert remaining == {ids[1], ids[3]} # only the pending rows survive diff --git a/tests/server/test_attendance.py b/tests/server/test_attendance.py index 10388888..375788e1 100644 --- a/tests/server/test_attendance.py +++ b/tests/server/test_attendance.py @@ -1466,7 +1466,7 @@ async def test_window_route_put_fixes_session(app): def _seed_parse_fight( - db_path, + schema, *, title="Trakanon", started_at, @@ -1476,13 +1476,11 @@ def _seed_parse_fight( world=_WORLD, hidden=False, ): - import sqlite3 as _sq - - with _sq.connect(db_path) as conn: + with pg_conn(schema) as conn: cur = conn.execute( "INSERT INTO encounters (world, act_encid, title, zone, started_at, ended_at, duration_s, " "success_level, source_dsn, uploaded_by, guild_name, ingested_at, hidden_at) " - "VALUES (?, ?, ?, 'Veeshan''s Peak', ?, ?, ?, 1, 'eq2act', 'Up', ?, ?, ?)", + "VALUES (%s, %s, %s, 'Veeshan''s Peak', %s, %s, %s, 1, 'eq2act', 'Up', %s, %s, %s) RETURNING id", ( world, f"enc-{started_at}-{title}", @@ -1495,13 +1493,12 @@ def _seed_parse_fight( started_at if hidden else None, ), ) - eid = int(cur.lastrowid or 0) + eid = int(cur.fetchone()["id"]) for name in players: conn.execute( - "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (?, ?, 1, 1)", + "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (%s, %s, 1, 1)", (eid, name), ) - conn.commit() return eid diff --git a/tests/server/test_erasure.py b/tests/server/test_erasure.py index fb4c1b86..a0bd705d 100644 --- a/tests/server/test_erasure.py +++ b/tests/server/test_erasure.py @@ -8,8 +8,6 @@ import base64 import json -import sqlite3 -from pathlib import Path from unittest.mock import patch import itsdangerous @@ -17,8 +15,8 @@ from httpx import ASGITransport, AsyncClient from psycopg.types.json import Json +from backend.server.db import erasure as erasure_mod from backend.server.db.erasure import DELETED_SOURCE_DSN, DELETED_USER_ID, erase_user_sync -from backend.server.parses import db as pdb from tests.fixtures.pg import pg_conn from tests.fixtures.users import make_fake_admin @@ -36,11 +34,10 @@ def users_db(users_schema: str) -> str: @pytest.fixture -def parses_db(tmp_path, monkeypatch) -> Path: - db = tmp_path / "parses.db" - monkeypatch.setattr(pdb.store, "path", db) - pdb.ParsesStore(db).init_db().close() - return db +def parses_db(parses_db_path: str) -> str: + """Leased parses schema — erasure reads ``parses.db.SCHEMA`` at call + time, and the ``parses_db_path`` fixture already re-points it.""" + return parses_db_path def _seed_users(schema: str) -> None: @@ -116,39 +113,39 @@ def _seed_users(schema: str) -> None: ) -def _seed_parses(db: Path) -> None: - with sqlite3.connect(db) as c: +def _seed_parses(schema: str) -> None: + with pg_conn(schema) as c: for i, (dsn, hidden_by) in enumerate( ((f"plugin:{VICTIM}", None), (f"plugin:{OTHER}", VICTIM), (f"plugin:{OTHER}", OTHER)), start=1 ): c.execute( "INSERT INTO encounters (world, act_encid, title, zone, started_at, ended_at, duration_s, success_level, " "source_dsn, uploaded_by, guild_name, ingested_at, hidden_at, hidden_by) " - "VALUES ('Varsoon', ?, 'Tarinax', 'Deathtoll', ?, ?, 60, 1, ?, 'Sihtric', 'Exordium', ?, ?, ?)", + "VALUES ('Varsoon', %s, 'Tarinax', 'Deathtoll', %s, %s, 60, 1, %s, 'Sihtric', 'Exordium', %s, %s, %s)", (f"enc{i}", NOW, NOW + 60, dsn, NOW, NOW if hidden_by else None, hidden_by), ) c.execute( - "INSERT INTO ingest_log (world, act_encid, encounter_id, ingested_at, source_dsn) VALUES ('Varsoon', 'enc1', 1, ?, ?)", + "INSERT INTO ingest_log (world, act_encid, encounter_id, ingested_at, source_dsn) VALUES ('Varsoon', 'enc1', 1, %s, %s)", (NOW, f"plugin:{VICTIM}"), ) c.execute( "INSERT INTO tamper_reports (world, act_encid, title, started_at, ended_at, duration_s, uploader_logger_name, uploader_discord_id, uploader_discord_name, " - "guild_name, reason, reported_at, payload_json) VALUES ('Varsoon', 'enc1', 'Tarinax', 1800000000, 1800000060, 60, 'Sihtric', ?, 'Victim', 'Exordium', " - "'stale_encounter', ?, '{}')", + "guild_name, reason, reported_at, payload_json) VALUES ('Varsoon', 'enc1', 'Tarinax', 1800000000, 1800000060, 60, 'Sihtric', %s, 'Victim', 'Exordium', " + "'stale_encounter', %s, '{}')", (VICTIM, NOW), ) c.execute( "INSERT INTO tamper_reports (world, act_encid, title, started_at, ended_at, duration_s, uploader_logger_name, uploader_discord_id, uploader_discord_name, " - "guild_name, reason, reported_at, payload_json) VALUES ('Varsoon', 'enc2', 'Tarinax', 1800000000, 1800000060, 60, 'Alt', ?, 'Other', 'Exordium', " - "'stale_encounter', ?, '{}')", + "guild_name, reason, reported_at, payload_json) VALUES ('Varsoon', 'enc2', 'Tarinax', 1800000000, 1800000060, 60, 'Alt', %s, 'Other', 'Exordium', " + "'stale_encounter', %s, '{}')", (OTHER, NOW), ) -def _count(db: Path, sql: str, *params) -> int: - """parses.db (still SQLite) — ``?`` placeholders.""" - with sqlite3.connect(db) as c: - return c.execute(sql, params).fetchone()[0] +def _count(schema: str, sql: str, *params) -> int: + """parses schema (Postgres) — ``%s`` placeholders, COUNT aliased AS n.""" + with pg_conn(schema) as c: + return c.execute(sql, params).fetchone()["n"] def _count_u(schema: str, sql: str, *params) -> int: @@ -166,7 +163,7 @@ def test_erase_removes_owned_rows_tombstones_authorship_and_strips_uploads(users _seed_users(users_db) _seed_parses(parses_db) - result = erase_user_sync(VICTIM, users_schema=users_db, parses_path=parses_db, now=NOW) + result = erase_user_sync(VICTIM, users_schema=users_db, parses_schema=parses_db, now=NOW) assert result.found is True # Owned rows gone; the other user's rows intact. @@ -209,25 +206,25 @@ def test_erase_removes_owned_rows_tombstones_authorship_and_strips_uploads(users uploaders = c.execute("SELECT uploaders FROM attendance_sessions").fetchone()["uploaders"] assert uploaders == {OTHER: NOW} assert result.sessions_scrubbed == 1 - # parses.db: uploads stay, identity stripped; hidden_by cleared; reports gone. - assert _count(parses_db, "SELECT COUNT(*) FROM encounters") == 3 - assert _count(parses_db, "SELECT COUNT(*) FROM encounters WHERE source_dsn = ?", DELETED_SOURCE_DSN) == 1 - assert _count(parses_db, "SELECT COUNT(*) FROM encounters WHERE source_dsn = ?", f"plugin:{OTHER}") == 2 - assert _count(parses_db, "SELECT COUNT(*) FROM encounters WHERE hidden_by = ?", VICTIM) == 0 - assert _count(parses_db, "SELECT COUNT(*) FROM encounters WHERE hidden_by = ?", OTHER) == 1 - assert _count(parses_db, "SELECT COUNT(*) FROM ingest_log WHERE source_dsn = ?", DELETED_SOURCE_DSN) == 1 - assert _count(parses_db, "SELECT COUNT(*) FROM tamper_reports WHERE uploader_discord_id = ?", VICTIM) == 0 - assert _count(parses_db, "SELECT COUNT(*) FROM tamper_reports WHERE uploader_discord_id = ?", OTHER) == 1 + # parses schema: uploads stay, identity stripped; hidden_by cleared; reports gone. + assert _count(parses_db, "SELECT COUNT(*) AS n FROM encounters") == 3 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM encounters WHERE source_dsn = %s", DELETED_SOURCE_DSN) == 1 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM encounters WHERE source_dsn = %s", f"plugin:{OTHER}") == 2 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM encounters WHERE hidden_by = %s", VICTIM) == 0 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM encounters WHERE hidden_by = %s", OTHER) == 1 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM ingest_log WHERE source_dsn = %s", DELETED_SOURCE_DSN) == 1 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM tamper_reports WHERE uploader_discord_id = %s", VICTIM) == 0 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM tamper_reports WHERE uploader_discord_id = %s", OTHER) == 1 assert result.parses_anonymised == 1 assert result.tamper_reports_deleted == 1 def test_erase_is_idempotent_and_never_touches_the_tombstone(users_db, parses_db): _seed_users(users_db) - assert erase_user_sync(VICTIM, users_schema=users_db, parses_path=parses_db, now=NOW).found is True - again = erase_user_sync(VICTIM, users_schema=users_db, parses_path=parses_db, now=NOW) + assert erase_user_sync(VICTIM, users_schema=users_db, parses_schema=parses_db, now=NOW).found is True + again = erase_user_sync(VICTIM, users_schema=users_db, parses_schema=parses_db, now=NOW) assert again.found is False - assert erase_user_sync(DELETED_USER_ID, users_schema=users_db, parses_path=parses_db).found is False + assert erase_user_sync(DELETED_USER_ID, users_schema=users_db, parses_schema=parses_db).found is False assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", DELETED_USER_ID) == 1 @@ -290,7 +287,7 @@ async def test_self_service_erase_requires_typed_username_and_clears_session(app assert audit.call_args.args[0] == "user_erased" assert audit.call_args.kwargs["self_service"] is True assert _count_u(users_db, "SELECT COUNT(*) AS n FROM users WHERE discord_id = %s", VICTIM) == 0 - assert _count(parses_db, "SELECT COUNT(*) FROM encounters WHERE source_dsn = ?", DELETED_SOURCE_DSN) == 1 + assert _count(parses_db, "SELECT COUNT(*) AS n FROM encounters WHERE source_dsn = %s", DELETED_SOURCE_DSN) == 1 async def test_self_service_erase_requires_login(app): diff --git a/tests/server/test_export.py b/tests/server/test_export.py index 4b094144..3cddef82 100644 --- a/tests/server/test_export.py +++ b/tests/server/test_export.py @@ -7,7 +7,6 @@ from __future__ import annotations -from pathlib import Path from unittest.mock import AsyncMock, patch import pytest @@ -29,17 +28,13 @@ def users_db(users_schema: str) -> str: @pytest.fixture -def isolated_parses_db(tmp_path, monkeypatch: pytest.MonkeyPatch) -> Path: - """A private parses DB for the ingest-backed e2e test. Writing to the - session-shared parses DB shifts encounter ids for every later test — - the parses-list tests' lazy is_player backfill resolves mocked ids - against the real DB, so a stray extra row breaks them at a distance.""" - from backend.server.parses import db as parses_db - - p = tmp_path / "parses.db" - monkeypatch.setattr(parses_db, "DB_PATH", p) - monkeypatch.setattr(parses_db.store, "path", p) - return p +def isolated_parses_db(parses_db_path: str) -> str: + """A private leased parses schema for the ingest-backed e2e test. + Writing to the session-shared parses schema shifts encounter ids for + every later test — the parses-list tests' lazy is_player backfill + resolves mocked ids against the real DB, so a stray extra row breaks + them at a distance.""" + return parses_db_path def _grant_api(users_db: str, discord_id: str = "discord-123") -> None: diff --git a/tests/server/test_parses_delete.py b/tests/server/test_parses_delete.py index 04ad1e22..d8ebc8d7 100644 --- a/tests/server/test_parses_delete.py +++ b/tests/server/test_parses_delete.py @@ -368,10 +368,11 @@ async def test_delete_batch_caps_at_max_ids(app): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: r = await client.delete("/api/parses/batch?ids=" + ",".join(map(str, ids))) assert r.status_code == 200 - # First execute() is the auth SELECT: [*ids, world] — exactly the cap + world. - select_params = conn.execute.call_args_list[0].args[1] - assert len(select_params) == PARSE_BATCH_MAX_IDS + 1 - assert select_params[:PARSE_BATCH_MAX_IDS] == ids[:PARSE_BATCH_MAX_IDS] + # First execute() is the auth SELECT: (ids via = ANY, world) — exactly the cap. + ids_param, world_param = conn.execute.call_args_list[0].args[1] + assert len(ids_param) == PARSE_BATCH_MAX_IDS + assert ids_param == ids[:PARSE_BATCH_MAX_IDS] + assert world_param == "Varsoon" @pytest.mark.asyncio @@ -430,16 +431,14 @@ async def test_delete_batch_audit_names_the_guilds(app): @pytest.mark.asyncio -async def test_list_excludes_hidden_rows(app, tmp_path, monkeypatch): - # Real temp DB: one visible boss kill, one soft-deleted. +async def test_list_excludes_hidden_rows(app, parses_db_path): + # Real leased parses schema: one visible boss kill, one soft-deleted. import time as _t from backend.server.parses import db as pdb from backend.server.parses.models import Encounter - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(pdb.store, "path", db_file) - conn = pdb.ParsesStore(db_file).init_db() + conn = pdb.store.init_db() for encid, title in [("AAA", "Tarinax"), ("BBB", "Venekor")]: enc = Encounter( encid=encid, diff --git a/tests/server/test_parses_ingest.py b/tests/server/test_parses_ingest.py index 0f834aae..554d8d5d 100644 --- a/tests/server/test_parses_ingest.py +++ b/tests/server/test_parses_ingest.py @@ -139,7 +139,7 @@ async def test_ingest_revived_status_schedules_background(app): @pytest.mark.asyncio -async def test_reupload_of_soft_deleted_parse_does_not_revive_it(tmp_path, monkeypatch): +async def test_reupload_of_soft_deleted_parse_does_not_revive_it(parses_db_path): """Security: re-uploading a moderator-hidden parse must NOT un-hide it. Soft-delete is the moderation action; auto-reviving on re-upload let a @@ -151,19 +151,15 @@ async def test_reupload_of_soft_deleted_parse_does_not_revive_it(tmp_path, monke from backend.server.api.parses.ingest import _ingest_payload_sync from backend.server.parses import db as pdb - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(pdb.store, "path", db_file) - # init the schema - pdb.ParsesStore(db_file).init_db().close() - payload = IngestRequest(**_minimal_payload()) - # First ingest → inserted. + # First ingest → inserted (the fixture re-pointed the shared store at a + # leased scratch schema). status, eid, *_ = _ingest_payload_sync(payload, "Menludiir", "Exordium", "plugin:123", {}) assert status == "inserted" and eid is not None # Soft-delete it (as the delete route does for a boss kill). - conn = pdb.ParsesStore(db_file).init_db() + conn = pdb.store.init_db() try: pdb.store.soft_delete_encounter(conn, eid, hidden_at=1700000000) assert pdb.store.find_encounter_by_act_encid(conn, payload.encounter.encid)["hidden_at"] is not None @@ -175,7 +171,7 @@ async def test_reupload_of_soft_deleted_parse_does_not_revive_it(tmp_path, monke assert status2 == "skipped" assert eid2 is None # Still hidden — moderation preserved. - conn = pdb.ParsesStore(db_file).init_db() + conn = pdb.store.init_db() try: assert pdb.store.find_encounter_by_act_encid(conn, payload.encounter.encid)["hidden_at"] is not None finally: diff --git a/tests/server/test_parses_ingest_classify.py b/tests/server/test_parses_ingest_classify.py index c6d85038..89441f9c 100644 --- a/tests/server/test_parses_ingest_classify.py +++ b/tests/server/test_parses_ingest_classify.py @@ -7,9 +7,8 @@ from __future__ import annotations -import sqlite3 from datetime import datetime -from pathlib import Path +from typing import Any from unittest.mock import patch import pytest @@ -83,71 +82,30 @@ def _snap(*, cls=None, level=None, guild_name=None, ilvl=None) -> CombatantSnaps return CombatantSnapshot(level=level, guild_name=guild_name, cls=cls, ilvl=ilvl) -def _encounter_id_for(conn: sqlite3.Connection, act_encid: str) -> int: - row = conn.execute("SELECT id FROM encounters WHERE act_encid = ?", (act_encid,)).fetchone() - return int(row[0]) - - -class _NoCloseConn: - """Thin proxy around a sqlite3.Connection that makes close() a no-op. - - _update_snapshots_sync opens a connection via parses_db.store.init_db() and - always calls conn.close() in its finally block. By wrapping the shared - in-memory connection in this proxy, we keep the underlying connection - alive for assertions after the helper returns. - - All attribute access is forwarded to the real connection so the proxy - is transparent to every other sqlite3 API (execute, commit, row_factory, - etc.). - """ - - def __init__(self, real: sqlite3.Connection) -> None: - object.__setattr__(self, "_real", real) - - def close(self) -> None: # suppress — keep the in-memory DB alive - pass - - def real_close(self) -> None: - object.__getattribute__(self, "_real").close() - - def __getattr__(self, name: str): - return getattr(object.__getattribute__(self, "_real"), name) - - def __setattr__(self, name: str, value) -> None: - setattr(object.__getattribute__(self, "_real"), name, value) - - def __enter__(self): - return object.__getattribute__(self, "_real").__enter__() - - def __exit__(self, *args): - return object.__getattribute__(self, "_real").__exit__(*args) +def _encounter_id_for(conn: Any, act_encid: str) -> int: + row = conn.execute("SELECT id FROM encounters WHERE act_encid = %s", (act_encid,)).fetchone() + return int(row["id"]) @pytest.fixture -def parses_db_in_memory(monkeypatch, tmp_path): - """Point parses_db.DB_PATH at an in-memory DB for the duration of the test. - - Both _insert_encounter_rows_sync (caller-supplied conn) and - _update_snapshots_sync (opens its own via parses_db.ParsesStore(DB_PATH).init_db()) - need to share state, so we hand the same connection back across both - calls. - - _update_snapshots_sync calls conn.close() in its finally block. We wrap - the real connection in _NoCloseConn so close() is a no-op and the shared - in-memory connection stays alive for assertions after the helper returns. +def parses_conn(parses_db_path): + """Test-owned connection to the leased scratch parses schema. + + ``parses_db_path`` has already re-pointed the shared store, so + _insert_encounter_rows_sync (caller-supplied conn, commits at the end) + and _update_snapshots_sync (checks out + closes its own connection via + parses_db.store.init_db()) both target the same schema; committed writes + are visible across connections (READ COMMITTED), so the old SQLite + shared-connection/_NoCloseConn trick is no longer needed. """ - db_file = tmp_path / "parses.db" - real_conn = parses_db.ParsesStore(db_file).init_db() - conn = _NoCloseConn(real_conn) - monkeypatch.setattr(parses_db.store, "path", db_file) - monkeypatch.setattr(parses_db.store, "init_db", lambda *a, **k: conn) + conn = parses_db.store.init_db() try: yield conn finally: - conn.real_close() + conn.close() -def test_ingest_populates_is_player_on_every_combatant(parses_db_in_memory): +def test_ingest_populates_is_player_on_every_combatant(parses_conn): """Synthesise an ingest call directly (skip HTTP) and confirm is_player flows onto every combatant row.""" from backend.server.api.parses.ingest import _insert_encounter_rows_sync @@ -167,7 +125,7 @@ def test_ingest_populates_is_player_on_every_combatant(parses_db_in_memory): with patch("backend.server.api.parses.ingest._classify_zone", return_value="dungeon"): _insert_encounter_rows_sync( - parses_db_in_memory, + parses_conn, enc, combatants=combatants, damage_types=[], @@ -179,20 +137,20 @@ def test_ingest_populates_is_player_on_every_combatant(parses_db_in_memory): world="Varsoon", ) - rows = list(parses_db_in_memory.execute("SELECT name, ally, is_player FROM combatants ORDER BY id")) - by_name = {r[0]: r for r in rows} + rows = parses_conn.execute("SELECT name, ally, is_player FROM combatants ORDER BY id").fetchall() + by_name = {r["name"]: r for r in rows} # Stage 5 — cls set → player - assert by_name["Alpha"][2] == 1, by_name["Alpha"] + assert by_name["Alpha"]["is_player"] == 1, by_name["Alpha"] # Stage 3 — multi-word → pet - assert by_name["a krait warrior"][2] == 0, by_name["a krait warrior"] + assert by_name["a krait warrior"]["is_player"] == 0, by_name["a krait warrior"] # Stage 4 — regex match → pet - assert by_name["Gibab"][2] == 0, by_name["Gibab"] + assert by_name["Gibab"]["is_player"] == 0, by_name["Gibab"] # Enemy (ally=0) → classifier omits, is_player stays NULL - assert by_name["Test Boss"][2] is None, by_name["Test Boss"] + assert by_name["Test Boss"]["is_player"] is None, by_name["Test Boss"] -def test_async_snapshot_fill_reclassifies(parses_db_in_memory): +def test_async_snapshot_fill_reclassifies(parses_conn): """After ingest, an async snapshot fill updates cls. The classifier must re-run so is_player can flip from 0 → 1 for a previously- unconfirmed combatant that Census just resolved.""" @@ -204,7 +162,7 @@ def test_async_snapshot_fill_reclassifies(parses_db_in_memory): with patch("backend.server.api.parses.ingest._classify_zone", return_value="other"): _insert_encounter_rows_sync( - parses_db_in_memory, + parses_conn, enc, combatants=combatants, damage_types=[], @@ -217,16 +175,17 @@ def test_async_snapshot_fill_reclassifies(parses_db_in_memory): ) # All five start as is_player=0 because no cls + no bucket-fill in 'other' n_total=5. - baseline = {r[0]: r[1] for r in parses_db_in_memory.execute("SELECT name, is_player FROM combatants")} + baseline = {r["name"]: r["is_player"] for r in parses_conn.execute("SELECT name, is_player FROM combatants")} assert all(v == 0 for v in baseline.values()), baseline - # Simulate async fill: cls becomes known for P1 + P2. - enc_id = _encounter_id_for(parses_db_in_memory, "testenc2") + # Simulate async fill: cls becomes known for P1 + P2. (_update_snapshots_sync + # opens — and commits + closes — its own connection via the re-pointed store.) + enc_id = _encounter_id_for(parses_conn, "testenc2") snapshots = {"P1": _snap(cls="Wizard"), "P2": _snap(cls="Berserker")} with patch("backend.server.api.parses.ingest._classify_zone", return_value="other"): _update_snapshots_sync(enc_id, snapshots) - after = {r[0]: r[1] for r in parses_db_in_memory.execute("SELECT name, is_player FROM combatants")} + after = {r["name"]: r["is_player"] for r in parses_conn.execute("SELECT name, is_player FROM combatants")} assert after["P1"] == 1 assert after["P2"] == 1 assert after["P3"] == 0 diff --git a/tests/server/test_parses_ingest_client_warnings.py b/tests/server/test_parses_ingest_client_warnings.py index ec407c79..797c166c 100644 --- a/tests/server/test_parses_ingest_client_warnings.py +++ b/tests/server/test_parses_ingest_client_warnings.py @@ -15,7 +15,6 @@ from __future__ import annotations import json -from pathlib import Path import pytest @@ -26,40 +25,34 @@ def _read_client_warnings(conn, encounter_id: int) -> str | None: """Return the raw client_warnings column for a row.""" row = conn.execute( - "SELECT client_warnings FROM encounters WHERE id = ?", + "SELECT client_warnings FROM encounters WHERE id = %s", (encounter_id,), ).fetchone() - return None if row is None else row[0] + return None if row is None else row["client_warnings"] -def _ingest_and_get(payload: dict, *, tmp_path, monkeypatch) -> tuple[int, Path]: - """Drive ``_ingest_payload_sync`` against a fresh tmp DB and return - (encounter_id, db_file). The shared store's ``path`` is monkeypatched - to db_file below, so both the ingest write and the read-back go - through the same redirected DB; reads construct ``ParsesStore(db_file)`` - where they want to be explicit. - """ +def _ingest_and_get(payload: dict) -> int: + """Drive ``_ingest_payload_sync`` against the leased scratch schema (the + ``parses_db_path`` fixture has already re-pointed the shared store) and + return the new encounter id. Reads go through ``parses_db.store.init_db()`` + which targets the same schema.""" from backend.server.api.parses import IngestRequest from backend.server.api.parses.ingest import _ingest_payload_sync - db_file: Path = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - req = IngestRequest(**payload) status, eid, *_ = _ingest_payload_sync(req, "Menludiir", "Exordium", "plugin:123", {}) assert status == "inserted" assert eid is not None - return eid, db_file + return eid -def test_client_warnings_persisted_as_json(tmp_path, monkeypatch): +def test_client_warnings_persisted_as_json(parses_db_path): payload = _minimal_payload() payload["client_warnings"] = ["folder_hint_mismatch"] - eid, db_file = _ingest_and_get(payload, tmp_path=tmp_path, monkeypatch=monkeypatch) + eid = _ingest_and_get(payload) - conn = parses_db.ParsesStore(db_file).init_db() + conn = parses_db.store.init_db() try: raw = _read_client_warnings(conn, eid) finally: @@ -70,7 +63,7 @@ def test_client_warnings_persisted_as_json(tmp_path, monkeypatch): assert decoded == ["folder_hint_mismatch"] -def test_client_warnings_absent_leaves_column_null(tmp_path, monkeypatch): +def test_client_warnings_absent_leaves_column_null(parses_db_path): """The plugin omits the key entirely when there's nothing to flag. That's the resting state for non-tampered uploads — column stays NULL so the admin UI can use a single `warnings?.length` check to @@ -78,9 +71,9 @@ def test_client_warnings_absent_leaves_column_null(tmp_path, monkeypatch): payload = _minimal_payload() assert "client_warnings" not in payload # baseline - eid, db_file = _ingest_and_get(payload, tmp_path=tmp_path, monkeypatch=monkeypatch) + eid = _ingest_and_get(payload) - conn = parses_db.ParsesStore(db_file).init_db() + conn = parses_db.store.init_db() try: raw = _read_client_warnings(conn, eid) finally: @@ -89,16 +82,16 @@ def test_client_warnings_absent_leaves_column_null(tmp_path, monkeypatch): assert raw is None -def test_client_warnings_empty_list_stored_as_null(tmp_path, monkeypatch): +def test_client_warnings_empty_list_stored_as_null(parses_db_path): """An empty array round-trips to NULL, NOT to "[]". Either form means "no warnings" semantically, but NULL is the canonical resting state and lets the admin column-filter just check IS NULL / IS NOT NULL.""" payload = _minimal_payload() payload["client_warnings"] = [] - eid, db_file = _ingest_and_get(payload, tmp_path=tmp_path, monkeypatch=monkeypatch) + eid = _ingest_and_get(payload) - conn = parses_db.ParsesStore(db_file).init_db() + conn = parses_db.store.init_db() try: raw = _read_client_warnings(conn, eid) finally: @@ -107,7 +100,7 @@ def test_client_warnings_empty_list_stored_as_null(tmp_path, monkeypatch): assert raw is None -def test_client_warnings_sanitises_entries(tmp_path, monkeypatch): +def test_client_warnings_sanitises_entries(parses_db_path): """Defence-in-depth sanitisation at storage time. The plugin already enforces these caps client-side; we re-enforce on ingest so a tampered build of the plugin can't blow them past: @@ -125,9 +118,9 @@ def test_client_warnings_sanitises_entries(tmp_path, monkeypatch): "x" * 100, # over-long ] - eid, db_file = _ingest_and_get(payload, tmp_path=tmp_path, monkeypatch=monkeypatch) + eid = _ingest_and_get(payload) - conn = parses_db.ParsesStore(db_file).init_db() + conn = parses_db.store.init_db() try: raw = _read_client_warnings(conn, eid) finally: @@ -140,7 +133,7 @@ def test_client_warnings_sanitises_entries(tmp_path, monkeypatch): assert decoded == ["folder_hint_mismatch", "x" * 64] -def test_client_warnings_pydantic_rejects_oversized_list(tmp_path, monkeypatch): +def test_client_warnings_pydantic_rejects_oversized_list(): """The Pydantic validator on IngestRequest caps the list at 32 entries — a malformed/hostile payload doesn't reach the storage layer at all. Pin that ValidationError raises so the cap survives future refactors.""" diff --git a/tests/server/test_parses_ingest_duplicates.py b/tests/server/test_parses_ingest_duplicates.py index 9dd9abef..ce47b8ae 100644 --- a/tests/server/test_parses_ingest_duplicates.py +++ b/tests/server/test_parses_ingest_duplicates.py @@ -1,18 +1,18 @@ """Duplicate-key payloads on /api/parses/ingest. Before 2026-09-27 a payload with two attack_types rows for one -(attacker, swingtype, type) key reached SQLite as a UNIQUE violation and +(attacker, swingtype, type) key reached the DB as a UNIQUE violation and came back as a 500 + traceback. One third-party client retried that every ~2 s for nine hours. Now: identical repeats collapse, conflicting repeats -are a 422 naming the keys, and any other IntegrityError is a 422 too. +are a 422 naming the keys, and any other UniqueViolation is a 422 too. """ from __future__ import annotations import copy -import sqlite3 from unittest.mock import AsyncMock, patch +import psycopg import pytest from httpx import ASGITransport, AsyncClient @@ -24,7 +24,7 @@ _damage_types_from_payload, _ingest_payload_sync, ) -from backend.server.parses import db as pdb +from tests.fixtures.pg import pg_conn from tests.server._parses_ingest_fixtures import ( _fake_require_user, _minimal_payload, @@ -92,27 +92,16 @@ def test_message_caps_the_listed_keys_at_five(): # --------------------------------------------------------------------------- -@pytest.fixture -def parses_tmp_db(tmp_path, monkeypatch): - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(pdb.store, "path", db_file) - pdb.ParsesStore(db_file).init_db().close() - return db_file +def _encounter_count(schema: str) -> int: + with pg_conn(schema) as conn: + return conn.execute("SELECT COUNT(*) AS n FROM encounters").fetchone()["n"] -def _encounter_count(db_file) -> int: - conn = sqlite3.connect(db_file) - try: - return conn.execute("SELECT COUNT(*) FROM encounters").fetchone()[0] - finally: - conn.close() - - -def test_sync_ingest_rejects_conflicting_duplicates_and_writes_nothing(parses_tmp_db): +def test_sync_ingest_rejects_conflicting_duplicates_and_writes_nothing(parses_db_path): payload = IngestRequest(**_payload_with_duplicate_attack(conflicting=True)) with pytest.raises(DuplicatePayloadRows): _ingest_payload_sync(payload, "Menludiir", "Exordium", "plugin:123", {}) - assert _encounter_count(parses_tmp_db) == 0 + assert _encounter_count(parses_db_path) == 0 # The encid was never marked ingested either, so a corrected re-upload works. fixed = IngestRequest(**_minimal_payload(encid="DUPL0001")) status, eid, _n_c, _n_dt, n_at = _ingest_payload_sync(fixed, "Menludiir", "Exordium", "plugin:123", {}) @@ -120,7 +109,7 @@ def test_sync_ingest_rejects_conflicting_duplicates_and_writes_nothing(parses_tm assert n_at == 2 -def test_sync_ingest_collapses_identical_duplicates(parses_tmp_db): +def test_sync_ingest_collapses_identical_duplicates(parses_db_path): payload = IngestRequest(**_payload_with_duplicate_attack(conflicting=False)) status, eid, _n_c, _n_dt, n_at = _ingest_payload_sync(payload, "Menludiir", "Exordium", "plugin:123", {}) assert status == "inserted" and eid is not None @@ -133,7 +122,7 @@ def test_sync_ingest_collapses_identical_duplicates(parses_tmp_db): @pytest.mark.asyncio -async def test_route_returns_422_for_conflicting_duplicates(app, parses_tmp_db, caplog): +async def test_route_returns_422_for_conflicting_duplicates(app, parses_db_path, caplog): with ( patch("backend.server.api.parses.ingest.require_user_session_or_token", _fake_require_user), patch("backend.server.api.parses.ingest._resolve_uploader_guild_async", new=AsyncMock(return_value="Exordium")), @@ -148,7 +137,7 @@ async def test_route_returns_422_for_conflicting_duplicates(app, parses_tmp_db, detail = r.json()["detail"] assert "attack_types" in detail and "Menludiir/2/Smite" in detail assert "retrying" in detail - assert _encounter_count(parses_tmp_db) == 0 + assert _encounter_count(parses_db_path) == 0 rejected = [m for m in caplog.messages if "rejected malformed payload" in m] assert len(rejected) == 1 assert "EQ2AdvancedDesktop/1.25.77" in rejected[0] and "discord-123" in rejected[0] @@ -162,7 +151,9 @@ async def test_route_maps_any_other_integrity_error_to_422(app): patch("backend.server.api.parses.ingest._resolve_combatant_snapshots", new=AsyncMock(return_value={})), patch( "backend.server.api.parses.ingest._ingest_payload_sync", - side_effect=sqlite3.IntegrityError("UNIQUE constraint failed: something.else"), + side_effect=psycopg.errors.UniqueViolation( + 'duplicate key value violates unique constraint "something.else"' + ), ), ): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: diff --git a/tests/server/test_parses_list.py b/tests/server/test_parses_list.py index 4291942b..6b15aceb 100644 --- a/tests/server/test_parses_list.py +++ b/tests/server/test_parses_list.py @@ -388,19 +388,17 @@ def _seed_search_db(conn, world="Varsoon"): conn.execute( "INSERT INTO encounters (world, act_encid, title, zone, started_at, ended_at, " "duration_s, success_level, source_dsn, uploaded_by, guild_name, ingested_at) " - "VALUES (?, ?, ?, ?, ?, ?, 60, 1, 'eq2act', ?, 'Exordium', ?)", + "VALUES (%s, %s, %s, %s, %s, %s, 60, 1, 'eq2act', %s, 'Exordium', %s)", (world, f"enc-{i}", title, zone, 1_900_000_000 + i, 1_900_000_060 + i, uploader, 1_900_000_000 + i), ) conn.commit() -def test_list_encounters_search_filters_title_zone_uploader(tmp_path, monkeypatch): +def test_list_encounters_search_filters_title_zone_uploader(parses_db_path): from backend.server.api.parses.list import _list_encounters_sync from backend.server.parses import db as parses_db - db_path = tmp_path / "parses.db" - monkeypatch.setattr(parses_db.store, "path", db_path) - conn = parses_db.ParsesStore(db_path).init_db() + conn = parses_db.store.init_db() try: _seed_search_db(conn) finally: @@ -435,17 +433,21 @@ def test_detail_ability_dps_uses_fight_duration(parses_db_path): try: conn.execute( "INSERT INTO encounters (id, world, act_encid, title, started_at, ended_at, duration_s, " - "source_dsn, ingested_at) VALUES (1, 'Varsoon', 'E1', 'Tarinax', 0, 434, 434, 'eq2act', 1)" + "source_dsn, ingested_at) OVERRIDING SYSTEM VALUE " + "VALUES (1, 'Varsoon', 'E1', 'Tarinax', 0, 434, 434, 'eq2act', 1)" + ) + conn.execute( + "INSERT INTO combatants (id, encounter_id, name, ally, is_player) OVERRIDING SYSTEM VALUE " + "VALUES (10, 1, 'Fiix', 1, 1)" ) - conn.execute("INSERT INTO combatants (id, encounter_id, name, ally, is_player) VALUES (10, 1, 'Fiix', 1, 1)") conn.execute( "INSERT INTO attack_types (combatant_id, swing_type, attack_name, damage, dps, hits, swings) " - "VALUES (10, ?, ?, 1040928, 3884.0, 701, 701)", + "VALUES (10, %s, %s, 1040928, 3884.0, 701, 701)", (int(SwingType.NONMELEE), "Larval Outbreak"), ) conn.execute( "INSERT INTO attack_types (combatant_id, swing_type, attack_name, damage, dps, hits, swings) " - "VALUES (10, ?, ?, 86800, 999.0, 10, 10)", + "VALUES (10, %s, %s, 86800, 999.0, 10, 10)", (int(SwingType.HEAL), "Bria's Ballad"), ) conn.execute( diff --git a/tests/server/test_parses_pagination.py b/tests/server/test_parses_pagination.py index e2e15fe1..3ca156a7 100644 --- a/tests/server/test_parses_pagination.py +++ b/tests/server/test_parses_pagination.py @@ -16,6 +16,7 @@ from httpx import ASGITransport, AsyncClient from backend.server.parses import db as parses_db_mod +from tests.fixtures.pg import pg_conn from tests.fixtures.users import make_fake_require_user, make_fake_user from tests.server._parses_fixtures import _FAKE_ENCOUNTER @@ -103,21 +104,19 @@ async def test_before_param_reaches_the_row_query(app): def _seed_encounter(conn, enc_id: int, started_at: int, *, hidden_at: int | None = None) -> None: conn.execute( "INSERT INTO encounters (id, world, act_encid, title, started_at, ended_at, duration_s," - " source_dsn, ingested_at, hidden_at) VALUES (?, 'Varsoon', ?, ?, ?, ?, 60, 'act://x', ?, ?)", + " source_dsn, ingested_at, hidden_at) OVERRIDING SYSTEM VALUE" + " VALUES (%s, 'Varsoon', %s, %s, %s, %s, 60, 'act://x', %s, %s)", (enc_id, f"enc{enc_id}", f"Boss {enc_id}", started_at, started_at + 60, started_at, hidden_at), ) @pytest.fixture -def seeded_parses_db(tmp_path, monkeypatch): - store = parses_db_mod.ParsesStore(tmp_path / "parses.db") - monkeypatch.setattr(parses_db_mod.store, "path", store.path) - conn = store.init_db() - for enc_id, ts in ((1, 3000), (2, 2000), (3, 1000)): - _seed_encounter(conn, enc_id, ts) - conn.commit() - conn.close() - return store +def seeded_parses_db(parses_db_path: str) -> str: + """Leased parses schema (store already re-pointed) with three encounters.""" + with pg_conn(parses_db_path) as conn: + for enc_id, ts in ((1, 3000), (2, 2000), (3, 1000)): + _seed_encounter(conn, enc_id, ts) + return parses_db_path def test_list_encounters_sync_before_filters_older_rows(seeded_parses_db): @@ -131,7 +130,7 @@ def test_list_encounters_sync_before_filters_older_rows(seeded_parses_db): def test_admin_list_before_cursor(seeded_parses_db): - conn = seeded_parses_db.init_db() + conn = parses_db_mod.store.init_db() try: page1 = parses_db_mod.ParsesStore.list_encounters_for_admin(conn, world="Varsoon", limit=2) assert [r["id"] for r in page1] == [1, 2] diff --git a/tests/server/test_parses_pet_filter.py b/tests/server/test_parses_pet_filter.py index fa3b36a3..b9fd6c55 100644 --- a/tests/server/test_parses_pet_filter.py +++ b/tests/server/test_parses_pet_filter.py @@ -9,8 +9,7 @@ from __future__ import annotations -import sqlite3 -from pathlib import Path +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -23,54 +22,46 @@ @pytest.fixture -def parses_db_in_memory(tmp_path, monkeypatch): - """Point parses_db.DB_PATH at a tmp-path-backed DB. The schema is - pre-initialised by opening + closing a connection; subsequent - ``parses_db.store.init_db()`` calls (from the route handler in another - thread) open a fresh connection against the same file, which is - safe with sqlite3 + WAL. - - We can't use ``:memory:`` here because the route handler's read - path early-returns when ``DB_PATH.exists()`` is False, and - ``Path(':memory:').exists()`` is always False.""" - db_file = tmp_path / "backend.server.parses.db" - parses_db.ParsesStore(db_file).init_db().close() - monkeypatch.setattr(parses_db.store, "path", db_file) - # The test body needs an open connection to seed + assert against. - # The route handler opens its OWN connection via the unpatched - # init_db(path) — same file, separate handle, no thread issues. - conn = parses_db.ParsesStore(db_file).init_db() +def parses_conn(parses_db_path): + """Test-owned connection to the leased scratch parses schema. + + ``parses_db_path`` has already re-pointed the shared store, so the route + handler (running in the executor thread) checks out its OWN connection + against the same schema — the test commits its seeds so the route sees + them (READ COMMITTED).""" + conn = parses_db.store.init_db() try: yield conn finally: conn.close() -def _insert_encounter(conn: sqlite3.Connection, act_encid: str, zone: str = "Z") -> int: - cur = conn.execute( +def _insert_encounter(conn: Any, act_encid: str, zone: str = "Z") -> int: + row = conn.execute( """ INSERT INTO encounters ( act_encid, title, zone, started_at, ended_at, duration_s, total_damage, encdps, kills, deaths, source_dsn, ingested_at, world - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id """, (act_encid, "Test", zone, 1, 2, 1, 100, 100.0, 0, 0, "test", 1, "Varsoon"), - ) - return int(cur.lastrowid or 0) + ).fetchone() + return int(row["id"]) @pytest.mark.asyncio -async def test_player_count_reads_is_player_flag(app, parses_db_in_memory): +async def test_player_count_reads_is_player_flag(app, parses_conn): """A combatant whose is_player=0 must NOT count toward player_count even if its name is single-word + ally=1 (the old heuristic would have counted it).""" - enc_id = _insert_encounter(parses_db_in_memory, "encA") + enc_id = _insert_encounter(parses_conn, "encA") for name in ("Alpha", "Bravo", "Charlie"): - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (?, ?, ?, ?)", + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (%s, %s, %s, %s)", (enc_id, name, 1, 0), ) - parses_db_in_memory.commit() + parses_conn.commit() with patch("backend.server.api.parses.list._require_user", _fake_user): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: @@ -82,59 +73,59 @@ async def test_player_count_reads_is_player_flag(app, parses_db_in_memory): assert data["results"][0]["player_count"] == 0, "single-word allies with is_player=0 must not count" -def test_ensure_classified_backfills_null_rows(parses_db_in_memory): +def test_ensure_classified_backfills_null_rows(parses_conn): """Inserting an encounter with combatant.is_player=NULL (mimicking a historic pre-migration row) then calling _ensure_classified on it must populate is_player on every row.""" from backend.server.api.parses.list import _ensure_classified - enc_id = _insert_encounter(parses_db_in_memory, "encB", zone="Halls of Fate") + enc_id = _insert_encounter(parses_conn, "encB", zone="Halls of Fate") # Two confirmed (cls set) and one multi-word pet, all is_player=NULL. - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, cls, is_player) VALUES (?, ?, ?, ?, ?)", + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, cls, is_player) VALUES (%s, %s, %s, %s, %s)", (enc_id, "Alpha", 1, "Wizard", None), ) - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, cls, is_player) VALUES (?, ?, ?, ?, ?)", + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, cls, is_player) VALUES (%s, %s, %s, %s, %s)", (enc_id, "Bravo", 1, "Wizard", None), ) - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, cls, is_player) VALUES (?, ?, ?, ?, ?)", + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, cls, is_player) VALUES (%s, %s, %s, %s, %s)", (enc_id, "a krait warrior", 1, None, None), ) - parses_db_in_memory.commit() + parses_conn.commit() # Sanity: every is_player is NULL. - nulls = parses_db_in_memory.execute("SELECT COUNT(*) FROM combatants WHERE is_player IS NULL").fetchone()[0] + nulls = parses_conn.execute("SELECT COUNT(*) AS n FROM combatants WHERE is_player IS NULL").fetchone()["n"] assert nulls == 3 with patch("backend.server.api.parses.list._classify_zone", return_value="dungeon"): - _ensure_classified(parses_db_in_memory, enc_id, "Halls of Fate") + _ensure_classified(parses_conn, enc_id, "Halls of Fate") - rows = {r[0]: r[1] for r in parses_db_in_memory.execute("SELECT name, is_player FROM combatants")} + rows = {r["name"]: r["is_player"] for r in parses_conn.execute("SELECT name, is_player FROM combatants")} assert rows["Alpha"] == 1 assert rows["Bravo"] == 1 assert rows["a krait warrior"] == 0 -def test_ensure_classified_is_noop_when_already_classified(parses_db_in_memory): +def test_ensure_classified_is_noop_when_already_classified(parses_conn): """Once every combatant has is_player populated, _ensure_classified must not re-run the classifier (no extra writes).""" from backend.server.api.parses.list import _ensure_classified - enc_id = _insert_encounter(parses_db_in_memory, "encC") - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (?, ?, ?, ?)", + enc_id = _insert_encounter(parses_conn, "encC") + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (%s, %s, %s, %s)", (enc_id, "Alpha", 1, 1), ) - parses_db_in_memory.commit() + parses_conn.commit() with patch("backend.server.parses.pet_detection.classify_combatants") as fake: - _ensure_classified(parses_db_in_memory, enc_id, None) + _ensure_classified(parses_conn, enc_id, None) fake.assert_not_called() -def test_ensure_classified_ignores_enemy_null_rows(parses_db_in_memory): +def test_ensure_classified_ignores_enemy_null_rows(parses_conn): """Enemy rows keep is_player NULL FOREVER by design (classify_combatants omits ally != 1 from its result), so their NULLs must not trip the unclassified probe. Before the ally-scoped probe, every encounter with an @@ -143,38 +134,38 @@ def test_ensure_classified_ignores_enemy_null_rows(parses_db_in_memory): the rankings load and collided with ingest ("database is locked").""" from backend.server.api.parses.list import _ensure_classified - enc_id = _insert_encounter(parses_db_in_memory, "encD") + enc_id = _insert_encounter(parses_conn, "encD") # Allies fully classified; the enemy row NULL (its permanent state). - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (?, ?, ?, ?)", + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (%s, %s, %s, %s)", (enc_id, "Alpha", 1, 1), ) - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (?, ?, ?, ?)", + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (%s, %s, %s, %s)", (enc_id, "Zylphax the Shredder", 0, None), ) - parses_db_in_memory.commit() + parses_conn.commit() with patch("backend.server.parses.pet_detection.classify_combatants") as fake: - assert _ensure_classified(parses_db_in_memory, enc_id, None) is False + assert _ensure_classified(parses_conn, enc_id, None) is False fake.assert_not_called() # And the enemy row's NULL is untouched. - row = parses_db_in_memory.execute("SELECT is_player FROM combatants WHERE name = 'Zylphax the Shredder'").fetchone() - assert row[0] is None + row = parses_conn.execute("SELECT is_player FROM combatants WHERE name = 'Zylphax the Shredder'").fetchone() + assert row["is_player"] is None @pytest.mark.asyncio -async def test_phase4_merger_top_n_uses_is_player(app, parses_db_in_memory): +async def test_phase4_merger_top_n_uses_is_player(app, parses_conn): """Phase-4-of-parse-grouping-redo merger's top-N gate must filter on is_player=1, so a bucket-promoted player CAN appear in top-N for merge decisions and a regex-matched pet CANNOT (even with high encdps). Two uploads of the same fight with identical top-N should merge.""" for encid, uploader in (("encD1", "Alpha"), ("encD2", "Bravo")): - enc_id = _insert_encounter(parses_db_in_memory, encid) + enc_id = _insert_encounter(parses_conn, encid) # Override the guild/uploader on the freshly-inserted encounter row. - parses_db_in_memory.execute( - "UPDATE encounters SET uploaded_by = ?, guild_name = ?, title = ? WHERE id = ?", + parses_conn.execute( + "UPDATE encounters SET uploaded_by = %s, guild_name = %s, title = %s WHERE id = %s", (uploader, "Exordium", "Bossy", enc_id), ) # Top-3 by encdps in both encounters are the same three players — @@ -186,11 +177,11 @@ async def test_phase4_merger_top_n_uses_is_player(app, parses_db_in_memory): ("Charlie", 3000.0, 1), ("Gibab", 99999.0, 0), # pet — must not show in top-N ): - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, encdps, is_player) VALUES (?, ?, ?, ?, ?)", + parses_conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, encdps, is_player) VALUES (%s, %s, %s, %s, %s)", (enc_id, name, 1, encdps, is_player), ) - parses_db_in_memory.commit() + parses_conn.commit() with patch("backend.server.api.parses.list._require_user", _fake_user): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: diff --git a/tests/server/test_parses_tamper_report.py b/tests/server/test_parses_tamper_report.py index 7b78f8f7..c2c5fa3d 100644 --- a/tests/server/test_parses_tamper_report.py +++ b/tests/server/test_parses_tamper_report.py @@ -28,6 +28,14 @@ ) +@pytest.fixture(autouse=True) +def _parses_schema(parses_db_path: str) -> str: + """Isolated leased parses schema per test — the fixture re-points the + shared store, so the route's argless ``store.init_db()`` and the helpers + below hit the same scratch schema.""" + return parses_db_path + + def _signed_with_reason( payload: dict, *, @@ -58,16 +66,16 @@ def _read_encounter_count() -> int: the whole reason this endpoint exists separately.""" conn = parses_db.store.init_db() try: - row = conn.execute("SELECT COUNT(*) FROM encounters").fetchone() - return int(row[0]) if row else 0 + row = conn.execute("SELECT COUNT(*) AS n FROM encounters").fetchone() + return int(row["n"]) if row else 0 finally: conn.close() def _wipe_tamper_reports() -> None: - """Per-test isolation. The session-scoped tmp DB is shared across - tests, but each tamper-report test starts with an empty table so - assertion counts don't drift across runs.""" + """Belt-and-braces isolation: the leased scratch schema already starts + empty per test, but an explicit wipe keeps assertion counts honest even + if a future fixture change shares state again.""" conn = parses_db.store.init_db() try: conn.execute("DELETE FROM tamper_reports") diff --git a/tests/server/test_parses_world_scoping.py b/tests/server/test_parses_world_scoping.py index 43fbe537..e16e1011 100644 --- a/tests/server/test_parses_world_scoping.py +++ b/tests/server/test_parses_world_scoping.py @@ -12,8 +12,6 @@ import hashlib import hmac import json -from dataclasses import replace -from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -21,8 +19,7 @@ from backend.server.api.parses import IngestRequest from backend.server.api.parses.ingest import _ingest_payload_sync -from backend.server.parses import db as parses_db -from backend.server.parses.models import Encounter +from tests.fixtures.pg import pg_conn from tests.fixtures.users import make_fake_admin _fake_admin_user = make_fake_admin(id="admin1") @@ -89,49 +86,31 @@ def _minimal_payload(encid: str = "ABCD1234") -> dict: class TestIngestPayloadSyncWorldAttribution: - def test_logger_server_wuoshi_stores_under_wuoshi(self, tmp_path, monkeypatch): + def test_logger_server_wuoshi_stores_under_wuoshi(self, parses_db_path): """When ingest is called with world='Wuoshi', the encounter row has world='Wuoshi'.""" - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - payload = IngestRequest(**_minimal_payload()) status, eid, *_ = _ingest_payload_sync(payload, "Menludiir", None, "plugin:123", {}, world="Wuoshi") assert status == "inserted" - conn = parses_db.ParsesStore(db_file).init_db() - try: - row = conn.execute("SELECT world FROM encounters WHERE id = ?", (eid,)).fetchone() - assert row[0] == "Wuoshi" - finally: - conn.close() + with pg_conn(parses_db_path) as conn: + row = conn.execute("SELECT world FROM encounters WHERE id = %s", (eid,)).fetchone() + assert row["world"] == "Wuoshi" - def test_ingest_log_world_matches_encounter(self, tmp_path, monkeypatch): + def test_ingest_log_world_matches_encounter(self, parses_db_path): """ingest_log.world must match the encounter's world.""" - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - payload = IngestRequest(**_minimal_payload()) _ingest_payload_sync(payload, "Menludiir", None, "plugin:123", {}, world="Kaladim") - conn = parses_db.ParsesStore(db_file).init_db() - try: + with pg_conn(parses_db_path) as conn: row = conn.execute( - "SELECT world FROM ingest_log WHERE act_encid = ?", (payload.encounter.encid,) + "SELECT world FROM ingest_log WHERE act_encid = %s", (payload.encounter.encid,) ).fetchone() - assert row[0] == "Kaladim" - finally: - conn.close() + assert row["world"] == "Kaladim" - def test_same_encid_different_world_both_inserted(self, tmp_path, monkeypatch): + def test_same_encid_different_world_both_inserted(self, parses_db_path): """Two ingest calls with the same act_encid but different worlds must both succeed (no UNIQUE collision).""" - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - payload = IngestRequest(**_minimal_payload()) status_v, eid_v, *_ = _ingest_payload_sync(payload, "Menludiir", None, "plugin:123", {}, world="Varsoon") status_w, eid_w, *_ = _ingest_payload_sync(payload, "Menludiir", None, "plugin:123", {}, world="Wuoshi") @@ -139,22 +118,15 @@ def test_same_encid_different_world_both_inserted(self, tmp_path, monkeypatch): assert status_w == "inserted" assert eid_v != eid_w - conn = parses_db.ParsesStore(db_file).init_db() - try: + with pg_conn(parses_db_path) as conn: count = conn.execute( - "SELECT COUNT(*) FROM encounters WHERE act_encid = ?", (payload.encounter.encid,) - ).fetchone()[0] - assert count == 2 - finally: - conn.close() + "SELECT COUNT(*) AS n FROM encounters WHERE act_encid = %s", (payload.encounter.encid,) + ).fetchone()["n"] + assert count == 2 - def test_idempotency_is_world_scoped(self, tmp_path, monkeypatch): + def test_idempotency_is_world_scoped(self, parses_db_path): """Re-uploading the same (world, act_encid) returns 'skipped'; uploading the same act_encid under a DIFFERENT world is NOT skipped.""" - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - payload = IngestRequest(**_minimal_payload()) _ingest_payload_sync(payload, "Menludiir", None, "plugin:123", {}, world="Varsoon") @@ -173,13 +145,9 @@ def test_idempotency_is_world_scoped(self, tmp_path, monkeypatch): class TestListEncountersSyncWorldScoping: - def test_varsoon_list_excludes_wuoshi_encounters(self, tmp_path, monkeypatch): + def test_varsoon_list_excludes_wuoshi_encounters(self, parses_db_path): """_list_encounters_sync(world='Varsoon') must not return encounters stored under 'Wuoshi'.""" - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - payload_v = IngestRequest(**_minimal_payload("VARSOON1")) payload_w = IngestRequest(**_minimal_payload("WUOSHI01")) @@ -213,14 +181,10 @@ def _fake_admin(self, request=None): return _fake_admin_user @pytest.mark.asyncio - async def test_cross_server_delete_by_id_blocked(self, tmp_path, monkeypatch, app): + async def test_cross_server_delete_by_id_blocked(self, parses_db_path, app): """Seeded: id A=Varsoon, id B=Wuoshi. DELETE /api/parses/{B} under Varsoon context → 404, B still in DB. DELETE /api/parses/{A} under Varsoon context → 200, A gone.""" - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - # Seed two encounters in different worlds. payload_a = IngestRequest(**_minimal_payload("AAAAAA")) payload_b = IngestRequest(**_minimal_payload("BBBBBB")) @@ -243,12 +207,9 @@ async def test_cross_server_delete_by_id_blocked(self, tmp_path, monkeypatch, ap assert r_same.json() == {"deleted": 1} # Verify B still exists in the DB; A is gone. - conn = parses_db.ParsesStore(db_file).init_db() - try: - b_row = conn.execute("SELECT id FROM encounters WHERE id = ?", (id_b,)).fetchone() - a_row = conn.execute("SELECT id FROM encounters WHERE id = ?", (id_a,)).fetchone() - finally: - conn.close() + with pg_conn(parses_db_path) as conn: + b_row = conn.execute("SELECT id FROM encounters WHERE id = %s", (id_b,)).fetchone() + a_row = conn.execute("SELECT id FROM encounters WHERE id = %s", (id_a,)).fetchone() assert b_row is not None, "Wuoshi encounter B must still exist after cross-server delete attempt" assert a_row is None, "Varsoon encounter A must be gone after same-server delete" @@ -328,7 +289,7 @@ def _http_payload(logger_server: str = "Wuoshi", encid: str = "E2E00001") -> dic @pytest.mark.asyncio -async def test_http_ingest_attributes_encounter_world_from_logger_server(tmp_path, monkeypatch, app): +async def test_http_ingest_attributes_encounter_world_from_logger_server(parses_db_path, app): """End-to-end: an HTTP POST to /api/parses/ingest with logger_server='Wuoshi' must result in encounters.world == 'Wuoshi'. @@ -337,10 +298,6 @@ async def test_http_ingest_attributes_encounter_world_from_logger_server(tmp_pat parse_world = sanitized_server wiring in ingest_parse that was introduced in the merge of feature/per-server-urls (replacing the old _resolve_parse_world + current_world() fallback).""" - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(parses_db.store, "path", db_file) - parses_db.ParsesStore(db_file).init_db().close() - token = "eq2c_e2e_test_token" async def _fake_user(request): @@ -379,14 +336,11 @@ async def _fake_user(request): encounter_id = r.json()["encounter_id"] assert encounter_id is not None - conn = parses_db.ParsesStore(db_file).init_db() - try: - row = conn.execute("SELECT world FROM encounters WHERE id = ?", (encounter_id,)).fetchone() - finally: - conn.close() + with pg_conn(parses_db_path) as conn: + row = conn.execute("SELECT world FROM encounters WHERE id = %s", (encounter_id,)).fetchone() assert row is not None, "encounter row not found in DB" - assert row[0] == "Wuoshi", ( - f"expected encounters.world='Wuoshi' but got {row[0]!r} — " + assert row["world"] == "Wuoshi", ( + f"expected encounters.world='Wuoshi' but got {row['world']!r} — " "the parse_world=sanitized_server wiring in ingest_parse is broken" ) diff --git a/tests/server/test_rankings.py b/tests/server/test_rankings.py index 6c387f31..f7f6195a 100644 --- a/tests/server/test_rankings.py +++ b/tests/server/test_rankings.py @@ -511,15 +511,13 @@ def _ins(conn, encid, title, *, success, players, guild, duration): # the DB layer). Stamp every ally as a player directly so the test # data lands at the pre-classifier intent (all 8 are players, not # 6 after the 'other' zone bucket-fill cap). - conn.execute("UPDATE combatants SET is_player = 1 WHERE encounter_id = ? AND ally = 1", (eid,)) + conn.execute("UPDATE combatants SET is_player = 1 WHERE encounter_id = %s AND ally = 1", (eid,)) conn.commit() @pytest.fixture() -def rankings_db(tmp_path, monkeypatch): - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(pdb.store, "path", db_file) - conn = pdb.ParsesStore(db_file).init_db() +def rankings_db(parses_db_path): + conn = pdb.store.init_db() _ins(conn, "WIN", "Tarinax", success=1, players=8, guild="Exordium", duration=60) # boss, raid _ins(conn, "TRASH", "a krait", success=1, players=8, guild="Exordium", duration=30) # not boss _ins(conn, "LOSS", "Cazel", success=2, players=8, guild="Exordium", duration=90) # not a win @@ -533,7 +531,7 @@ def rankings_db(tmp_path, monkeypatch): rk.rankings_cache.delete(f"{rk._KILLS_KEY}:{world}") # Also clear the legacy bare key in case it was left behind by an older run. rk.rankings_cache.delete(rk._KILLS_KEY) - return db_file + return parses_db_path def test_loader_keeps_only_winning_boss_kills(rankings_db): @@ -667,7 +665,7 @@ async def test_rankings_default_xpac_per_server(app, users_schema): @pytest.mark.asyncio -async def test_rankings_leaderboard_is_world_scoped(app, monkeypatch, tmp_path, users_schema): +async def test_rankings_leaderboard_is_world_scoped(app, users_schema, parses_db_path): """The /rankings leaderboard endpoint must return per-world data via the executor call path. @@ -688,10 +686,9 @@ async def test_rankings_leaderboard_is_world_scoped(app, monkeypatch, tmp_path, db.upsert_server_settings_sync("Wuoshi", max_level=70, current_xpac=None, launch_dt=None) server_context.load_registry() - # Seed a Varsoon boss kill and a distinct Wuoshi boss kill. - db_file = tmp_path / "backend.server.parses.db" - monkeypatch.setattr(pdb.store, "path", db_file) - conn = pdb.ParsesStore(db_file).init_db() + # Seed a Varsoon boss kill and a distinct Wuoshi boss kill (leased + # parses schema; the store is already re-pointed by parses_db_path). + conn = pdb.store.init_db() _ins(conn, "V-WIN", "Tarinax", success=1, players=8, guild="Exordium", duration=60) # defaults to Varsoon # Wuoshi kill uses a different boss title so results are unambiguous. enc_w = Encounter( @@ -763,7 +760,7 @@ async def test_rankings_leaderboard_is_world_scoped(app, monkeypatch, tmp_path, # rankings loader's is_player=1 player_count subquery yields 8 # (raid scope), not 6 after the 'other' zone bucket-fill cap. # Mirrors the same UPDATE in the _ins helper above. - conn.execute("UPDATE combatants SET is_player = 1 WHERE encounter_id = ? AND ally = 1", (weid,)) + conn.execute("UPDATE combatants SET is_player = 1 WHERE encounter_id = %s AND ally = 1", (weid,)) conn.commit() conn.close() @@ -895,16 +892,15 @@ def test_resolve_boss_curated_zone_rejects_unmatched_titles(): # --------------------------------------------------------------------------- -def _ins_enemy(db_file, title, name, deaths=1): - import sqlite3 as _sqlite3 +def _ins_enemy(schema, title, name, deaths=1): + from tests.fixtures.pg import pg_conn - with _sqlite3.connect(db_file) as conn: - eid = conn.execute("SELECT id FROM encounters WHERE title = ?", (title,)).fetchone()[0] + with pg_conn(schema) as conn: + eid = conn.execute("SELECT id FROM encounters WHERE title = %s", (title,)).fetchone()["id"] conn.execute( - "INSERT INTO combatants (encounter_id, name, ally, deaths) VALUES (?, ?, 0, ?)", + "INSERT INTO combatants (encounter_id, name, ally, deaths) VALUES (%s, %s, 0, %s)", (eid, name, deaths), ) - conn.commit() def test_missing_required_mobs(): diff --git a/tests/server/test_zones_cache_invalidation.py b/tests/server/test_zones_cache_invalidation.py index 76d2e4e8..4a35e97b 100644 --- a/tests/server/test_zones_cache_invalidation.py +++ b/tests/server/test_zones_cache_invalidation.py @@ -263,57 +263,46 @@ async def test_delete_mob_no_invalidation_on_404(app, editor_override) -> None: # Phase 5 — invalidate_zones_cache also nukes combatant is_player # --------------------------------------------------------------------------- -import sqlite3 -from pathlib import Path - from backend.server.parses import db as parses_db @pytest.fixture -def parses_db_in_memory(monkeypatch, tmp_path): - """Shared in-memory parses DB for the duration of the test. - - Patches invalidate_is_player_cache so it operates on the in-memory - connection rather than opening a new connection to DB_PATH.""" - db_file = tmp_path / "parses.db" - conn = parses_db.ParsesStore(db_file).init_db() - monkeypatch.setattr(parses_db.store, "path", db_file) - monkeypatch.setattr(parses_db.store, "init_db", lambda *a, **k: conn) - monkeypatch.setattr( - parses_db.store, - "invalidate_is_player_cache", - lambda *a, **k: parses_db.store.invalidate_is_player_cache_with_conn(conn), - ) +def parses_db_leased(parses_db_path): + """Open store connection to a leased scratch parses schema (the shared + store is already re-pointed, so invalidate_is_player_cache's own + connection lands in the same schema).""" + conn = parses_db.store.init_db() try: yield conn finally: conn.close() -def test_invalidate_zones_cache_nukes_combatant_is_player(parses_db_in_memory): +def test_invalidate_zones_cache_nukes_combatant_is_player(parses_db_leased): from backend.server.api.rankings import invalidate_zones_cache - cur = parses_db_in_memory.execute( + cur = parses_db_leased.execute( """ INSERT INTO encounters ( act_encid, title, zone, started_at, ended_at, duration_s, total_damage, encdps, kills, deaths, source_dsn, ingested_at, world - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id """, ("invZ", "Test", "Z", 1, 2, 1, 100, 100.0, 0, 0, "test", 1, "Varsoon"), ) - enc_id = int(cur.lastrowid or 0) - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (?, ?, ?, ?)", + enc_id = int(cur.fetchone()["id"]) + parses_db_leased.execute( + "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (%s, %s, %s, %s)", (enc_id, "Alpha", 1, 1), ) - parses_db_in_memory.execute( - "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (?, ?, ?, ?)", + parses_db_leased.execute( + "INSERT INTO combatants (encounter_id, name, ally, is_player) VALUES (%s, %s, %s, %s)", (enc_id, "Bravo", 1, 0), ) - parses_db_in_memory.commit() + parses_db_leased.commit() invalidate_zones_cache() - rows = parses_db_in_memory.execute("SELECT is_player FROM combatants").fetchall() - assert all(r[0] is None for r in rows), "every is_player must be NULL after invalidate_zones_cache" + rows = parses_db_leased.execute("SELECT is_player FROM combatants").fetchall() + assert all(r["is_player"] is None for r in rows), "every is_player must be NULL after invalidate_zones_cache" From 38c8ae188e79c450bd6e8a21780b7bdb28177285 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 16:35:26 +0100 Subject: [PATCH 20/32] feat(pg): reviewed zones + raids family schemas (0004, 0005) zones: builder-assigned ids stay plain bigint PKs; curator-managed encounter/mob rows get identity ids; featured-raid trio + _meta ported; flags stay 0/1 integers. raids: all ids identity (scrape+curator inserts); markdown blobs stay text; ACT trigger/spell-timer tables ported column-for-column (absolute_/fill_color quirks kept); strftime defaults -> extract(epoch). Both verified to apply cleanly via retarget to throwaway schemas on local PG17. Co-Authored-By: Claude Fable 5 --- db/migrations/0004_zones.sql | 114 ++++++++++++++++++++++ db/migrations/0005_raids.sql | 181 +++++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 db/migrations/0004_zones.sql create mode 100644 db/migrations/0005_raids.sql diff --git a/db/migrations/0004_zones.sql b/db/migrations/0004_zones.sql new file mode 100644 index 00000000..019162d0 --- /dev/null +++ b/db/migrations/0004_zones.sql @@ -0,0 +1,114 @@ +create schema if not exists zones; +set search_path to zones, public; + +-- Reviewed translation of backend/eq2db/zones.sql's schema blocks. +-- zones.id is BUILDER-ASSIGNED (scripts/build_zones_db.py seeds explicit +-- ids from the cleaned wiki JSON) — plain bigint PK, no identity. +-- zone_encounters / zone_encounter_mobs are curator-managed at runtime — +-- identity ids (the copy script preserves existing ids via OVERRIDING + +-- setval). 0/1 flags stay integers per the type policy. + +CREATE TABLE _meta ( + key text PRIMARY KEY, + value text +); + +CREATE TABLE zones ( + -- Identity + id bigint PRIMARY KEY, + name text NOT NULL UNIQUE, + name_lower text NOT NULL, + + -- Expansion attribution + expansion_short text NOT NULL, -- 'DoF', 'AoM', 'CoE', ... + expansion_name text NOT NULL, -- 'Desert of Flames', ... + expansion_year integer, + expansion_confidence text NOT NULL, -- 'category', 'live_update', ... + expansion_source text, -- audit trail / reason + + -- Flags + is_persistent_instance integer NOT NULL DEFAULT 0, + is_endless_persistent integer NOT NULL DEFAULT 0, + is_tradeskill integer NOT NULL DEFAULT 0, + is_pvp integer NOT NULL DEFAULT 0, + is_openworld integer NOT NULL DEFAULT 0, + is_instance integer NOT NULL DEFAULT 0, + is_live_event integer NOT NULL DEFAULT 0, + is_city integer NOT NULL DEFAULT 0, + is_contested integer NOT NULL DEFAULT 0, + is_deprecated integer NOT NULL DEFAULT 0, + + -- Optional metadata + event_name text, -- when is_live_event=1 + wiki_url text +); + +-- Many-to-many zone ↔ type. A zone with both Solo and Group variants gets +-- two rows so a "all group zones in RoK" query is one indexed JOIN. +CREATE TABLE zone_types ( + zone_id bigint NOT NULL REFERENCES zones(id) ON DELETE CASCADE, + type text NOT NULL, + PRIMARY KEY (zone_id, type) +); + +-- Alias → canonical zone. ACT logs may emit "The Fabled Deathtoll" or +-- "Fabled Deathtoll"; find_by_name checks aliases before failing. +CREATE TABLE zone_aliases ( + alias text NOT NULL PRIMARY KEY, + alias_lower text NOT NULL, + zone_id bigint NOT NULL REFERENCES zones(id) ON DELETE CASCADE +); + +-- Raid encounters per zone — hand-curated. Each row is a named encounter +-- (1 mob solo, or 2-4 mobs grouped). encounter_name is the display label; +-- individual mob names live in zone_encounter_mobs for reverse lookup. +CREATE TABLE zone_encounters ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + zone_id bigint NOT NULL REFERENCES zones(id) ON DELETE CASCADE, + encounter_name text NOT NULL, + position integer NOT NULL, + stage text, + wiki_url text, + UNIQUE (zone_id, position) +); + +CREATE TABLE zone_encounter_mobs ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + encounter_id bigint NOT NULL REFERENCES zone_encounters(id) ON DELETE CASCADE, + mob_name text NOT NULL, + mob_name_lower text NOT NULL, + position integer NOT NULL DEFAULT 0 +); + +CREATE TABLE featured_raid_expansions ( + expansion_short text PRIMARY KEY, + added_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) +); + +CREATE TABLE featured_raid_zones ( + zone_id bigint PRIMARY KEY REFERENCES zones(id) ON DELETE CASCADE, + added_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) +); + +CREATE TABLE featured_raid_categories ( + expansion_short text NOT NULL, + name text NOT NULL, + position integer NOT NULL, + PRIMARY KEY (expansion_short, name) +); + +CREATE INDEX idx_zones_name_lower ON zones (name_lower); +CREATE INDEX idx_zones_expansion ON zones (expansion_short); +CREATE INDEX idx_zones_event ON zones (is_live_event, event_name); +CREATE INDEX idx_zones_tradeskill ON zones (is_tradeskill); +CREATE INDEX idx_zone_types_type ON zone_types (type); +CREATE INDEX idx_zone_types_zone ON zone_types (zone_id); +CREATE INDEX idx_zone_aliases_lower ON zone_aliases (alias_lower); +CREATE INDEX idx_zone_aliases_zone ON zone_aliases (zone_id); +CREATE INDEX idx_zone_enc_zone ON zone_encounters (zone_id, position); +CREATE INDEX idx_zone_enc_mobs_enc ON zone_encounter_mobs (encounter_id, position); +CREATE INDEX idx_zone_enc_mobs_lower ON zone_encounter_mobs (mob_name_lower); + +-- seeds +-- (none — zone metadata is loaded by scripts/build_zones_db.py and boss +-- data is curator-managed; marker kept for the leaser's reset contract) diff --git a/db/migrations/0005_raids.sql b/db/migrations/0005_raids.sql new file mode 100644 index 00000000..815f0bf0 --- /dev/null +++ b/db/migrations/0005_raids.sql @@ -0,0 +1,181 @@ +create schema if not exists raids; +set search_path to raids, public; + +-- Reviewed translation of backend/eq2db/raids.sql's schema blocks. All ids +-- are runtime-generated (scrape ingest + curator edits) — identity columns; +-- the copy script preserves existing ids via OVERRIDING + setval. 0/1 flags +-- stay integers; epoch stamps bigint; markdown/text blobs stay text. + +CREATE TABLE _meta ( + key text PRIMARY KEY, + value text +); + +CREATE TABLE raid_zones ( + -- Identity + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + zone_name text NOT NULL UNIQUE, -- matches zones schema zones.name + zone_name_lower text NOT NULL, + + -- Denormalised from the zones family (intentional duplication so this + -- schema is queryable standalone; if the canonical changes, re-run the + -- scraper / sync job to refresh). + expansion_short text NOT NULL, -- 'Vanilla' / 'DoF' / 'KoS' / 'EoF' / 'RoK' + wiki_url text, + + -- Zone-level metadata extracted from the IZoneInformation template + -- on the wiki. All optional — missing fields just stay NULL. + access_md text, + background_md text, + overview_md text, + level_range text, -- e.g. '72-75' + zdiff text, -- 'x4' / 'x2' / 'x3' + lockout_min text, -- e.g. '2 days 20 hours' + lockout_max text, -- e.g. '7 days' + + -- Audit trail + source text NOT NULL, -- SOURCE_SCRAPE / SOURCE_MANUAL + last_synced_at bigint, + last_edited_at bigint, + last_edited_by text -- discord_id or 'eq2i_scrape' +); + +CREATE TABLE raid_encounters ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + raid_zone_id bigint NOT NULL REFERENCES raid_zones(id) ON DELETE CASCADE, + mob_name text NOT NULL, + mob_name_lower text NOT NULL, + position integer NOT NULL DEFAULT 0, -- order within the zone + + -- Free-form markdown strategy. Single blob deliberately — if a + -- structured pattern emerges (cures, dispels, phases) we can split + -- later without breaking callers. + strategy_md text, + + wiki_url text, + source text NOT NULL, + last_synced_at bigint, + last_edited_at bigint, + last_edited_by text, + + UNIQUE (raid_zone_id, mob_name_lower) +); + +CREATE TABLE raid_encounter_revisions ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + encounter_id bigint NOT NULL REFERENCES raid_encounters(id) ON DELETE CASCADE, + edited_at bigint NOT NULL, + edited_by text NOT NULL, -- discord_id or scrape token + before_md text, -- previous strategy_md (NULL on create) + after_md text NOT NULL, -- new strategy_md + edit_note text -- optional commit-message style note +); + +CREATE TABLE raid_zone_revisions ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + raid_zone_id bigint NOT NULL REFERENCES raid_zones(id) ON DELETE CASCADE, + edited_at bigint NOT NULL, + edited_by text NOT NULL, -- discord_id or scrape token + before_md text, -- NULL on the very first row (seed) + after_md text NOT NULL, + edit_note text -- optional commit-style note +); + +-- ACT Triggers — regex-driven matchers a player imports into Advanced Combat +-- Tracker to react to in-game log lines (boss callouts, debuffs, mechanic +-- triggers). One row maps 1:1 to a element in ACT's +-- spell_timers.xml export format (column names mirror XML attributes via +-- snake_case). +-- +-- A trigger with timer=1 references an entry in act_spell_timers by +-- timer_name; on XML export both rows are emitted so the dropped file +-- round-trips in ACT without manual fix-up. +CREATE TABLE act_triggers ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + raid_encounter_id bigint NOT NULL REFERENCES raid_encounters(id) ON DELETE CASCADE, + + -- Display / curation (web-only — no XML counterpart) + position integer NOT NULL DEFAULT 0, + label text, + notes text, + + -- ACT attributes (9 fields) + active integer NOT NULL DEFAULT 1, + regex text NOT NULL, + sound_data text NOT NULL DEFAULT '', + sound_type integer NOT NULL DEFAULT 3, -- 3 = TTS, 0 = silent / file + category_restrict integer NOT NULL DEFAULT 0, + category text, -- defaults to mob_name at write time + timer integer NOT NULL DEFAULT 0, + timer_name text, -- loose name-FK into act_spell_timers (same encounter) + tabbed integer NOT NULL DEFAULT 0, + + -- EQ2Parser enrichment (served by /api/act/pack — NEVER exported to + -- ACT XML; plain ACT stays byte-compatible) + cooldown_seconds double precision NOT NULL DEFAULT 1.0, + + -- Audit + last_edited_at bigint, + last_edited_by text, + created_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) +); + +-- ACT Spell Timers — named timer definitions referenced by act_triggers +-- via timer_name. One row maps 1:1 to a element in ACT's +-- spell_timers.xml. Multiple triggers MAY reference the same timer name +-- within an encounter (DRY); export deduplicates by name. +CREATE TABLE act_spell_timers ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + raid_encounter_id bigint NOT NULL REFERENCES raid_encounters(id) ON DELETE CASCADE, + + -- Identity (Name is what triggers reference via TimerName) + name text NOT NULL, + name_lower text NOT NULL, + + -- ACT attributes (17 fields) + checked integer NOT NULL DEFAULT 0, + timer_duration_s integer NOT NULL, -- "Timer" attribute in XML + only_master_ticks integer NOT NULL DEFAULT 0, + restrict integer NOT NULL DEFAULT 0, + absolute_ integer NOT NULL DEFAULT 0, -- "Absolute" — disambiguated from the SQL keyword + start_wav text NOT NULL DEFAULT '', + warning_wav text NOT NULL DEFAULT '', + warning_value integer NOT NULL DEFAULT 10, + radial_display integer NOT NULL DEFAULT 0, + modable integer NOT NULL DEFAULT 0, + tooltip text NOT NULL DEFAULT '', + fill_color integer NOT NULL DEFAULT -16776961, -- ACT default blue (.NET ARGB packed int) + panel1 integer NOT NULL DEFAULT 1, + panel2 integer NOT NULL DEFAULT 0, + remove_value integer NOT NULL DEFAULT -15, + category text, -- defaults to mob_name at write time + restrict_category integer NOT NULL DEFAULT 0, + + -- EQ2Parser enrichment (served by /api/act/pack — NEVER exported to + -- ACT XML; plain ACT stays byte-compatible). Closed vocabularies — + -- damage_type is comma-joined dominant-first log schools ("poison, + -- disease"); control_effect is tooltip vocabulary ("stifle", "stun"). + damage_type text NOT NULL DEFAULT '', + control_effect text NOT NULL DEFAULT '', + + -- Audit + last_edited_at bigint, + last_edited_by text, + created_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + + UNIQUE (raid_encounter_id, name_lower) +); + +CREATE INDEX idx_raid_zones_name_lower ON raid_zones (zone_name_lower); +CREATE INDEX idx_raid_zones_expansion ON raid_zones (expansion_short); +CREATE INDEX idx_raid_enc_zone ON raid_encounters (raid_zone_id, position); +CREATE INDEX idx_raid_enc_mob_lower ON raid_encounters (mob_name_lower); +CREATE INDEX idx_raid_rev_encounter ON raid_encounter_revisions (encounter_id, edited_at); +CREATE INDEX idx_raid_zone_rev_zone ON raid_zone_revisions (raid_zone_id, edited_at); +CREATE INDEX idx_act_triggers_enc ON act_triggers (raid_encounter_id, position); +CREATE INDEX idx_act_triggers_timer ON act_triggers (raid_encounter_id, timer_name); +CREATE INDEX idx_act_spell_timers_enc ON act_spell_timers (raid_encounter_id); + +-- seeds +-- (none — wiki-scrape seeded via scripts/dev/ingest_raids_json.py, then +-- curator-edited in place; marker kept for the leaser's reset contract) From 0a1e24720cf21f847b9f42cae5f47a78ed214a34 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 16:39:30 +0100 Subject: [PATCH 21/32] =?UTF-8?q?feat(pg):=20cutover=20copy=20script=20?= =?UTF-8?q?=E2=80=94=20introspected,=20FK-ordered,=20re-runnable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/migrate_to_postgres.py: per family one transaction (TRUNCATE RESTART IDENTITY CASCADE -> COPY in topo-sorted FK order -> setval identities -> verify counts -> COMMIT). Columns/types introspected from information_schema; transforms by type (days CSV -> integer[], jsonb validated + shipped as text, bytea passthrough). SQLite-side FK orphan anti-joins quarantine rows to reports/ and fail without --allow-orphans; duplicate approved claims abort before any write; the tiered detail retention is applied AT LOAD (attack/damage_types rows past their zone tier never reach Postgres; encounters stamped detail_pruned_at) — the step that turns 6.8GB of SQLite into the tier-gate measurement. Target host confirmation required (--yes to skip). JSON run report. Co-Authored-By: Claude Fable 5 --- scripts/migrate_to_postgres.py | 456 +++++++++++++++++++++++++++++++++ 1 file changed, 456 insertions(+) create mode 100644 scripts/migrate_to_postgres.py diff --git a/scripts/migrate_to_postgres.py b/scripts/migrate_to_postgres.py new file mode 100644 index 00000000..ba519578 --- /dev/null +++ b/scripts/migrate_to_postgres.py @@ -0,0 +1,456 @@ +"""One-shot SQLite → Postgres data copy for the Supabase cutover. + +Re-runnable by design: every invocation TRUNCATEs the target schemas and +reloads them, so rehearsals are the real run and the cutover window's +final pass is just a fast re-run against the last litestream restore. + + uv run python scripts/migrate_to_postgres.py \ + --users data/users.db --parses data/parses/parses.db \ + --census data/census/census.db --zones data/zones/zones.db \ + --raids data/raids/raids.db + +The DSN comes from the usual env chain (DATABASE_URL → SUPABASE_DB_URL → +POSTGRES_CONNECTION_STRING) or --dsn. Safety: the script prints the target +host and requires --yes (or interactive confirmation) before writing. + +Per family (one transaction each): TRUNCATE … RESTART IDENTITY CASCADE → +COPY each table in FK order → setval every identity sequence → COMMIT. +Column lists and types are introspected from information_schema, so the +copy follows the reviewed PG schemas; transforms applied by type: + + * integer[] (raid_slots.days) — "2,4" CSV → {2,4} + * jsonb (attendance zones/uploaders, census data_json) — validated + with json.loads, shipped as text (COPY parses jsonb input server-side) + * bytea (guild_recruitment.logo) — bytes passthrough + * detail-retention tiers (parses) — attack_types/damage_types rows + whose encounter is past its tier cutoff are NOT copied; the encounter + is stamped detail_pruned_at at load (this is what shrinks 6.8GB of + SQLite to a Postgres footprint the tier gate measures). + +FK orphans (users.db never enforced its FKs): each child table is +anti-joined against its parent IN SQLITE first; offending rows are written +to reports/orphans_..jsonl and EXCLUDED from the copy. Any +orphan fails the run unless --allow-orphans. Duplicate APPROVED claims per +(world, lower(name)) are audited the same way (the new partial unique +index would reject the load). + +Verification per table: PG row count must equal sqlite count minus +quarantined/pruned rows (hard fail), plus a sample deep-compare; a JSON +report lands in reports/migrate_report.json. +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_REPO)) + +from dotenv import load_dotenv # noqa: E402 + +load_dotenv(_REPO / ".env") + +from psycopg import conninfo # noqa: E402 + +from backend import pg # noqa: E402 + +REPORTS_DIR = _REPO / "reports" + +#: family schema → (cli flag name). Copy order matters only within a family. +FAMILIES = ("users", "parses", "census", "zones", "raids") + +#: Tables whose rows are intentionally NOT copied (none today; placeholder +#: so a future exclusion is one line). +SKIP_TABLES: set[str] = set() + +_DAY_S = 86_400 + + +# --------------------------------------------------------------------------- +# Introspection +# --------------------------------------------------------------------------- + + +def pg_tables(conn: Any, schema: str) -> list[str]: + rows = conn.execute( + "SELECT tablename FROM pg_tables WHERE schemaname = %s ORDER BY tablename", (schema,) + ).fetchall() + return [r["tablename"] for r in rows] + + +def pg_columns(conn: Any, schema: str, table: str) -> dict[str, str]: + """{column: data_type} — data_type is 'ARRAY'/'jsonb'/'bytea'/'bigint'/…""" + rows = conn.execute( + "SELECT column_name, data_type FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = %s ORDER BY ordinal_position", + (schema, table), + ).fetchall() + return {r["column_name"]: r["data_type"] for r in rows} + + +def pg_identity_columns(conn: Any, schema: str, table: str) -> list[str]: + rows = conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = %s AND is_identity = 'YES'", + (schema, table), + ).fetchall() + return [r["column_name"] for r in rows] + + +def pg_fks(conn: Any, schema: str) -> list[tuple[str, str, str, str]]: + """[(child_table, child_col, parent_table, parent_col)] within one schema.""" + rows = conn.execute( + """ + SELECT tc.table_name AS child, kcu.column_name AS child_col, + ccu.table_name AS parent, ccu.column_name AS parent_col + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage ccu + ON tc.constraint_name = ccu.constraint_name AND tc.table_schema = ccu.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = %s + """, + (schema,), + ).fetchall() + return [(r["child"], r["child_col"], r["parent"], r["parent_col"]) for r in rows] + + +def topo_order(tables: list[str], fks: list[tuple[str, str, str, str]]) -> list[str]: + """Parents before children (stable for independents).""" + deps: dict[str, set[str]] = {t: set() for t in tables} + for child, _cc, parent, _pc in fks: + if child != parent and parent in deps and child in deps: + deps[child].add(parent) + out: list[str] = [] + remaining = dict(deps) + while remaining: + ready = sorted(t for t, d in remaining.items() if not (d & set(remaining))) + if not ready: + raise RuntimeError(f"FK cycle among: {sorted(remaining)}") + out.extend(ready) + for t in ready: + remaining.pop(t) + return out + + +def sqlite_columns(sconn: sqlite3.Connection, table: str) -> list[str]: + return [r[1] for r in sconn.execute(f"PRAGMA table_info({table})").fetchall()] + + +def sqlite_table_exists(sconn: sqlite3.Connection, table: str) -> bool: + return sconn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone() is not None + + +# --------------------------------------------------------------------------- +# Transforms +# --------------------------------------------------------------------------- + + +def transform_value(value: Any, pg_type: str, *, table: str, column: str) -> Any: + if value is None: + return None + if pg_type == "ARRAY": + # raid_slots.days "2,4" → [2, 4] + if isinstance(value, str): + return [int(p) for p in value.split(",") if p.strip() != ""] + return value + if pg_type == "jsonb": + if isinstance(value, str): + json.loads(value) # validate; COPY ships the text and PG parses it + return value + return json.dumps(value) + if pg_type == "bytea": + return value if isinstance(value, (bytes, memoryview)) else bytes(value) + return value + + +# --------------------------------------------------------------------------- +# Audits +# --------------------------------------------------------------------------- + + +@dataclass +class TableReport: + source_rows: int = 0 + quarantined: int = 0 + pruned_detail: int = 0 + copied: int = 0 + pg_rows: int = 0 + + +@dataclass +class RunReport: + started_at: int = field(default_factory=lambda: int(time.time())) + tables: dict[str, TableReport] = field(default_factory=dict) + orphan_files: list[str] = field(default_factory=list) + duplicate_claims: int = 0 + detail_pruned_encounters: int = 0 + + def table(self, key: str) -> TableReport: + return self.tables.setdefault(key, TableReport()) + + +def quarantine_rows(schema: str, table: str, rows: list[dict], report: RunReport) -> None: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + path = REPORTS_DIR / f"orphans_{schema}.{table}.jsonl" + with path.open("a", encoding="utf-8") as fh: + for r in rows: + fh.write(json.dumps(r, default=str) + "\n") + report.orphan_files.append(str(path)) + + +def audit_duplicate_approved_claims(sconn: sqlite3.Connection, report: RunReport) -> list[str]: + """The 0001 partial unique index rejects >1 approved claim per + (world, lower(name)) — find them BEFORE the load fails mid-copy.""" + if not sqlite_table_exists(sconn, "character_claims"): + return [] + rows = sconn.execute( + "SELECT world, LOWER(character_name) AS n, COUNT(*) AS c FROM character_claims" + " WHERE status='approved' GROUP BY world, LOWER(character_name) HAVING COUNT(*) > 1" + ).fetchall() + report.duplicate_claims = len(rows) + return [f"{r[0]}/{r[1]} ×{r[2]}" for r in rows] + + +# --------------------------------------------------------------------------- +# Family copy +# --------------------------------------------------------------------------- + + +def copy_family( + conn: Any, + sconn: sqlite3.Connection, + schema: str, + report: RunReport, + *, + now: int, + prune_detail: bool, +) -> None: + conn.execute(pg.search_path_sql(schema)) + tables = [t for t in pg_tables(conn, schema) if t not in SKIP_TABLES] + fks = pg_fks(conn, schema) + order = topo_order(tables, fks) + + # Child-FK exclusion sets, built in sqlite (its FKs were never enforced). + fk_by_child: dict[str, list[tuple[str, str, str]]] = {} + for child, child_col, parent, parent_col in fks: + fk_by_child.setdefault(child, []).append((child_col, parent, parent_col)) + + # Parses detail tiers: compute once which encounter ids are past their + # tier (classification needs zones.db via the app's classifier). + pruned_encounters: set[int] = set() + if schema == "parses" and prune_detail and sqlite_table_exists(sconn, "encounters"): + from backend.server.parses.cleanup import detail_retention_days # noqa: PLC0415 + + for eid, zone, started_at in sconn.execute("SELECT id, zone, started_at FROM encounters"): + if started_at < now - detail_retention_days(zone) * _DAY_S: + pruned_encounters.add(int(eid)) + report.detail_pruned_encounters = len(pruned_encounters) + + sql_tables = ", ".join(f'"{t}"' for t in tables) + conn.execute(f"TRUNCATE {sql_tables} RESTART IDENTITY CASCADE") + + pruned_combatant_ids: set[int] = set() + if schema == "parses" and pruned_encounters and sqlite_table_exists(sconn, "combatants"): + ids = list(pruned_encounters) + for i in range(0, len(ids), 900): + chunk = ids[i : i + 900] + ph = ",".join("?" * len(chunk)) + for r in sconn.execute(f"SELECT id FROM combatants WHERE encounter_id IN ({ph})", chunk): + pruned_combatant_ids.add(int(r[0])) + + for table in order: + key = f"{schema}.{table}" + trep = report.table(key) + if not sqlite_table_exists(sconn, table): + if table == "_meta": + continue # census/zones/raids provenance may be absent — fine + print(f" [warn] sqlite has no table {table!r} — skipping (PG side stays empty)") + continue + + pg_cols = pg_columns(conn, schema, table) + s_cols = sqlite_columns(sconn, table) + cols = [c for c in pg_cols if c in s_cols] + missing_in_sqlite = [c for c in pg_cols if c not in s_cols] + if missing_in_sqlite: + print(f" [info] {key}: PG-only columns left at defaults: {missing_in_sqlite}") + + # Orphan audit for this child table. + excluded_where: list[str] = [] + for child_col, parent, parent_col in fk_by_child.get(table, []): + if not sqlite_table_exists(sconn, parent): + continue + orphans = [ + dict(zip(s_cols, row)) + for row in sconn.execute( + f"SELECT * FROM {table} WHERE {child_col} IS NOT NULL" + f" AND {child_col} NOT IN (SELECT {parent_col} FROM {parent})" + ).fetchall() + ] + if orphans: + quarantine_rows(schema, table, orphans, report) + trep.quarantined += len(orphans) + excluded_where.append(f"({child_col} IS NULL OR {child_col} IN (SELECT {parent_col} FROM {parent}))") + + where = (" WHERE " + " AND ".join(excluded_where)) if excluded_where else "" + select_sql = f"SELECT {', '.join(cols)} FROM {table}{where}" + trep.source_rows = sconn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + + col_types = [pg_cols[c] for c in cols] + detail_table = schema == "parses" and table in ("attack_types", "damage_types") + combatant_idx = cols.index("combatant_id") if detail_table and "combatant_id" in cols else None + + copy_sql = f"COPY {table} ({', '.join(cols)}) FROM STDIN" + copied = 0 + with conn.cursor() as cur, cur.copy(copy_sql) as cp: + for row in sconn.execute(select_sql): + if combatant_idx is not None and int(row[combatant_idx]) in pruned_combatant_ids: + trep.pruned_detail += 1 + continue + cp.write_row( + tuple(transform_value(v, t, table=table, column=c) for v, t, c in zip(row, col_types, cols)) + ) + copied += 1 + trep.copied = copied + + # Stamp detail_pruned_at on the pruned encounters once encounters landed. + if schema == "parses" and table == "encounters" and pruned_encounters: + conn.execute( + "UPDATE encounters SET detail_pruned_at = %s WHERE id = ANY(%s)", + (now, list(pruned_encounters)), + ) + + # setval every identity sequence to MAX(col). + for table in order: + for col in pg_identity_columns(conn, schema, table): + conn.execute( + "SELECT setval(pg_get_serial_sequence(%s, %s)," + f" COALESCE((SELECT MAX({col}) FROM {table}), 0) + 1, false)", + (f"{schema}.{table}", col), + ) + + # Verify counts inside the same transaction. + for table in order: + key = f"{schema}.{table}" + trep = report.table(key) + row = conn.execute(f"SELECT COUNT(*) AS n FROM {table}").fetchone() + trep.pg_rows = row["n"] + expected = trep.copied + if trep.pg_rows != expected: + raise RuntimeError(f"{key}: copied {expected} rows but PG holds {trep.pg_rows}") + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + for fam in FAMILIES: + ap.add_argument(f"--{fam}", type=Path, help=f"path to the source {fam} SQLite file") + ap.add_argument("--dsn", help="override the env DSN chain") + ap.add_argument("--allow-orphans", action="store_true", help="quarantine orphans and continue") + ap.add_argument("--no-detail-tiers", action="store_true", help="copy ALL attack/damage_types rows") + ap.add_argument("--yes", action="store_true", help="skip the interactive target confirmation") + args = ap.parse_args() + + if args.dsn: + import os + + os.environ["DATABASE_URL"] = args.dsn + params = conninfo.conninfo_to_dict(pg.dsn()) + target = f"{params.get('host', '?')}/{params.get('dbname', '?')}" + print(f"Target Postgres: {target}") + if not args.yes: + if input(f"TRUNCATE + reload the family schemas on {target}? [type 'yes'] ").strip() != "yes": + print("aborted") + return 1 + + sources = {fam: getattr(args, fam) for fam in FAMILIES if getattr(args, fam) is not None} + if not sources: + print("nothing to do — pass at least one -- path") + return 1 + for fam, path in sources.items(): + if not path.exists(): + print(f"--{fam}: {path} does not exist") + return 1 + + now = int(time.time()) + report = RunReport() + rc = 0 + + # Users first: audit duplicate approved claims before anything writes. + if "users" in sources: + with sqlite3.connect(sources["users"]) as sconn: + dups = audit_duplicate_approved_claims(sconn, report) + if dups: + print(f"DUPLICATE APPROVED CLAIMS ({len(dups)}) — resolve in prod before cutover:") + for d in dups: + print(f" {d}") + return 2 + + for fam in FAMILIES: + if fam not in sources: + continue + print(f"[{fam}] copying from {sources[fam]} ...") + t0 = time.time() + sconn = sqlite3.connect(f"file:{sources[fam]}?mode=ro", uri=True) + try: + with pg.connection() as conn: + copy_family( + conn, + sconn, + fam, + report, + now=now, + prune_detail=not args.no_detail_tiers, + ) + conn.commit() + finally: + sconn.close() + print(f"[{fam}] done in {time.time() - t0:.1f}s") + + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + out = REPORTS_DIR / "migrate_report.json" + out.write_text( + json.dumps( + { + "started_at": report.started_at, + "target": target, + "detail_pruned_encounters": report.detail_pruned_encounters, + "duplicate_claims": report.duplicate_claims, + "orphan_files": sorted(set(report.orphan_files)), + "tables": {k: vars(v) for k, v in sorted(report.tables.items())}, + }, + indent=2, + ), + encoding="utf-8", + ) + print(f"report: {out}") + + total_quarantined = sum(t.quarantined for t in report.tables.values()) + if total_quarantined: + print(f"quarantined {total_quarantined} orphan row(s) — see reports/orphans_*.jsonl") + if not args.allow_orphans: + print("failing (pass --allow-orphans to accept the quarantine)") + rc = 3 + for key, t in sorted(report.tables.items()): + print( + f" {key}: source={t.source_rows} copied={t.copied} pg={t.pg_rows}" + + (f" quarantined={t.quarantined}" if t.quarantined else "") + + (f" detail_pruned={t.pruned_detail}" if t.pruned_detail else "") + ) + return rc + + +if __name__ == "__main__": + pg.ensure_selector_event_loop_policy() + raise SystemExit(main()) From 33c4adcd23e84ed4d5e0a7701a882bf33fd65044 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 16:45:45 +0100 Subject: [PATCH 22/32] =?UTF-8?q?test(pg):=20census-dependent=20test=20fil?= =?UTF-8?q?es=20on=20the=20leased-schema=20fixtures=20=E2=80=94=20suite=20?= =?UTF-8?q?green?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eleven files converted by two agents + verified: census_schema replaces every store.path/DB_PATH re-point (the shared instance re-point makes route patches unnecessary), schema-shape tests became information_schema invariants, top_n seeds satisfy the now-enforced combatants FK via stub encounters, dead parses_db.path patches dropped from admin tests. Full suite: 2153 passed, 0 failed on local PostgreSQL 17. Co-Authored-By: Claude Fable 5 --- tests/census/test_store.py | 387 +++++++++++------------ tests/server/test_aa_census_store.py | 62 ++-- tests/server/test_admin_parses.py | 4 - tests/server/test_admin_servers.py | 2 - tests/server/test_character.py | 24 +- tests/server/test_gear_sets.py | 78 ++--- tests/server/test_guild.py | 29 +- tests/server/test_guild_history.py | 12 +- tests/server/test_guild_recruitment.py | 9 +- tests/server/test_parses_ingest_cache.py | 9 +- tests/server/test_parses_top_n.py | 40 +-- 11 files changed, 276 insertions(+), 380 deletions(-) diff --git a/tests/census/test_store.py b/tests/census/test_store.py index 82fb472c..cb5c2319 100644 --- a/tests/census/test_store.py +++ b/tests/census/test_store.py @@ -1,164 +1,153 @@ from __future__ import annotations -from backend.census import store as cs - +import pytest -def test_init_db_creates_tables(tmp_path): - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} - assert {"characters", "guilds"} <= tables - char_cols = {r[1] for r in conn.execute("PRAGMA table_info(characters)")} - assert { - "name_lower", - "world", - "name", - "level", - "guild_name", - "data_json", - "last_resolved_at", - "updated_at", - } <= char_cols - finally: - conn.close() +from backend.census import store as cs +from tests.fixtures.pg import pg_conn -def test_upsert_character_resolved_then_get(tmp_path): - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() +@pytest.fixture +def conn(census_schema): + """Open schema-scoped connection to a leased scratch census schema.""" + c = cs.CensusStore(census_schema).init_db() try: - cs.CensusStore.upsert_character( - conn, - "Menludiir", - "Varsoon", - {"name": "Menludiir", "level": 90, "guild_name": "Exordium", "cls": "Templar"}, - resolved=True, - now=1000, - ) - rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") - assert rec is not None - assert rec["data"]["level"] == 90 - assert rec["last_resolved_at"] == 1000 + yield c finally: - conn.close() + c.close() + + +def test_schema_has_tables(census_schema): + """Descendant of the SQLite init_db schema test: the migration-owned + census schema (db/migrations/0003_census.sql) carries the tables and + character columns the store relies on.""" + with pg_conn(census_schema) as conn: + rows = conn.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema = %s", + (census_schema,), + ).fetchall() + tables = {r["table_name"] for r in rows} + assert {"characters", "guilds"} <= tables + cols = conn.execute( + "SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = 'characters'", + (census_schema,), + ).fetchall() + char_cols = {r["column_name"] for r in cols} + assert { + "name_lower", + "world", + "name", + "level", + "guild_name", + "data_json", + "last_resolved_at", + "updated_at", + } <= char_cols + + +def test_upsert_character_resolved_then_get(conn): + cs.CensusStore.upsert_character( + conn, + "Menludiir", + "Varsoon", + {"name": "Menludiir", "level": 90, "guild_name": "Exordium", "cls": "Templar"}, + resolved=True, + now=1000, + ) + rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") + assert rec is not None + assert rec["data"]["level"] == 90 + assert rec["last_resolved_at"] == 1000 -def test_sparse_refresh_never_clobbers(tmp_path): - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - cs.CensusStore.upsert_character( - conn, "Menludiir", "Varsoon", {"name": "Menludiir", "level": 90, "cls": "Templar"}, resolved=True, now=1000 - ) - cs.CensusStore.upsert_character( - conn, "Menludiir", "Varsoon", {"name": "Menludiir", "level": None, "cls": None}, resolved=False, now=2000 - ) - rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") - assert rec is not None - assert rec["data"]["level"] == 90 # kept - assert rec["last_resolved_at"] == 1000 # unchanged - finally: - conn.close() +def test_sparse_refresh_never_clobbers(conn): + cs.CensusStore.upsert_character( + conn, "Menludiir", "Varsoon", {"name": "Menludiir", "level": 90, "cls": "Templar"}, resolved=True, now=1000 + ) + cs.CensusStore.upsert_character( + conn, "Menludiir", "Varsoon", {"name": "Menludiir", "level": None, "cls": None}, resolved=False, now=2000 + ) + rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") + assert rec is not None + assert rec["data"]["level"] == 90 # kept + assert rec["last_resolved_at"] == 1000 # unchanged -def test_roster_overview_does_not_wipe_resolved_gear(tmp_path): +def test_roster_overview_does_not_wipe_resolved_gear(conn): """A guild-roster overview (no equipment key) must NOT null an individually resolved character's gear — it overlays its scalar fields and preserves the rest, without advancing the freshness clock.""" - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - # Full individual resolve: gear + id present. - cs.CensusStore.upsert_character( - conn, - "Menludiir", - "Varsoon", - {"name": "Menludiir", "id": "123", "level": 70, "cls": "Templar", "equipment": [{"slot": "Head"}]}, - resolved=True, - now=1000, - ) - # Later guild-roster refresh: sparse overview, resolved=True, newer ts. - cs.CensusStore.upsert_character( - conn, - "Menludiir", - "Varsoon", - {"name": "Menludiir", "level": 71, "cls": "Templar", "deity": "Tunare"}, - resolved=True, - now=2000, - ) - rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") - assert rec is not None - assert rec["data"]["equipment"] == [{"slot": "Head"}] # gear preserved - assert rec["data"]["id"] == "123" # id preserved - assert rec["data"]["level"] == 71 # scalar refreshed from the overview - assert rec["data"]["deity"] == "Tunare" # new field merged in - assert rec["last_resolved_at"] == 1000 # partial overlay didn't advance freshness - finally: - conn.close() + # Full individual resolve: gear + id present. + cs.CensusStore.upsert_character( + conn, + "Menludiir", + "Varsoon", + {"name": "Menludiir", "id": "123", "level": 70, "cls": "Templar", "equipment": [{"slot": "Head"}]}, + resolved=True, + now=1000, + ) + # Later guild-roster refresh: sparse overview, resolved=True, newer ts. + cs.CensusStore.upsert_character( + conn, + "Menludiir", + "Varsoon", + {"name": "Menludiir", "level": 71, "cls": "Templar", "deity": "Tunare"}, + resolved=True, + now=2000, + ) + rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") + assert rec is not None + assert rec["data"]["equipment"] == [{"slot": "Head"}] # gear preserved + assert rec["data"]["id"] == "123" # id preserved + assert rec["data"]["level"] == 71 # scalar refreshed from the overview + assert rec["data"]["deity"] == "Tunare" # new field merged in + assert rec["last_resolved_at"] == 1000 # partial overlay didn't advance freshness -def test_full_resolve_replaces_and_advances_freshness(tmp_path): +def test_full_resolve_replaces_and_advances_freshness(conn): """A full resolve (equipment key present) overlays gear and advances the freshness clock — genuine gear changes still take effect.""" - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - cs.CensusStore.upsert_character( - conn, - "Menludiir", - "Varsoon", - {"name": "Menludiir", "id": "123", "equipment": [{"slot": "Head"}]}, - resolved=True, - now=1000, - ) - cs.CensusStore.upsert_character( - conn, - "Menludiir", - "Varsoon", - {"name": "Menludiir", "id": "123", "equipment": [{"slot": "Chest"}]}, - resolved=True, - now=2000, - ) - rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") - assert rec is not None - assert rec["data"]["equipment"] == [{"slot": "Chest"}] # replaced - assert rec["last_resolved_at"] == 2000 # advanced - finally: - conn.close() + cs.CensusStore.upsert_character( + conn, + "Menludiir", + "Varsoon", + {"name": "Menludiir", "id": "123", "equipment": [{"slot": "Head"}]}, + resolved=True, + now=1000, + ) + cs.CensusStore.upsert_character( + conn, + "Menludiir", + "Varsoon", + {"name": "Menludiir", "id": "123", "equipment": [{"slot": "Chest"}]}, + resolved=True, + now=2000, + ) + rec = cs.CensusStore.get_character(conn, "Menludiir", "Varsoon") + assert rec is not None + assert rec["data"]["equipment"] == [{"slot": "Chest"}] # replaced + assert rec["last_resolved_at"] == 2000 # advanced -def test_unresolved_first_sight_is_not_stored(tmp_path): - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - cs.CensusStore.upsert_character(conn, "Ghost", "Varsoon", {"name": "Ghost"}, resolved=False, now=1000) - assert cs.CensusStore.get_character(conn, "Ghost", "Varsoon") is None - finally: - conn.close() +def test_unresolved_first_sight_is_not_stored(conn): + cs.CensusStore.upsert_character(conn, "Ghost", "Varsoon", {"name": "Ghost"}, resolved=False, now=1000) + assert cs.CensusStore.get_character(conn, "Ghost", "Varsoon") is None -def test_get_missing_returns_none(tmp_path): - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - assert cs.CensusStore.get_character(conn, "Nobody", "Varsoon") is None - finally: - conn.close() +def test_get_missing_returns_none(conn): + assert cs.CensusStore.get_character(conn, "Nobody", "Varsoon") is None -def test_upsert_guild_then_get(tmp_path): - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - blob = {"name": "Exordium", "members": [{"name": "Menludiir", "rank": "Leader"}]} - cs.CensusStore.upsert_guild(conn, "Exordium", "Varsoon", blob, now=1000) - rec = cs.CensusStore.get_guild(conn, "Exordium", "Varsoon") - assert rec is not None - assert rec["data"]["members"][0]["name"] == "Menludiir" - assert rec["last_resolved_at"] == 1000 - finally: - conn.close() +def test_upsert_guild_then_get(conn): + blob = {"name": "Exordium", "members": [{"name": "Menludiir", "rank": "Leader"}]} + cs.CensusStore.upsert_guild(conn, "Exordium", "Varsoon", blob, now=1000) + rec = cs.CensusStore.get_guild(conn, "Exordium", "Varsoon") + assert rec is not None + assert rec["data"]["members"][0]["name"] == "Menludiir" + assert rec["last_resolved_at"] == 1000 -def test_guild_get_missing_returns_none(tmp_path): - conn = cs.CensusStore(tmp_path / "backend.census.db").init_db() - try: - assert cs.CensusStore.get_guild(conn, "Nope", "Varsoon") is None - finally: - conn.close() +def test_guild_get_missing_returns_none(conn): + assert cs.CensusStore.get_guild(conn, "Nope", "Varsoon") is None # --------------------------------------------------------------------------- @@ -169,10 +158,6 @@ def test_guild_get_missing_returns_none(tmp_path): _T0 = 1_800_000_000 # 2027-01-15T08:00:00Z -def _history_conn(tmp_path): - return cs.CensusStore(tmp_path / "backend.census.db").init_db() - - def _write(conn, now: int, level: int = 300, name: str = "Exordium", retention_days: int = 400) -> None: cs.CensusStore.upsert_guild_history( conn, @@ -191,71 +176,59 @@ def _write(conn, now: int, level: int = 300, name: str = "Exordium", retention_d ) -def test_init_db_creates_guild_history(tmp_path): - conn = _history_conn(tmp_path) - try: - cols = {r[1] for r in conn.execute("PRAGMA table_info(guild_history)")} - assert { - "world", - "name_lower", - "day", - "captured_at", - "level", - "members", - "accounts", - "achievement_count", - "max_level_members", - "distinct_classes", - } == cols - finally: - conn.close() - - -def test_same_day_writes_collapse_to_one_row_last_capture_wins(tmp_path): - conn = _history_conn(tmp_path) - try: - _write(conn, _T0, level=300) - _write(conn, _T0 + 3600, level=301) # 15-minute refresh later the same UTC day - rows = cs.CensusStore.get_guild_history(conn, "exordium", "Varsoon", 7, now=_T0 + 3600) - assert len(rows) == 1 - assert rows[0]["level"] == 301 - assert rows[0]["captured_at"] == _T0 + 3600 - assert rows[0]["day"] == cs.CensusStore._utc_day(_T0) - assert rows[0]["members"] == 40 and rows[0]["distinct_classes"] == 9 - finally: - conn.close() - - -def test_window_is_inclusive_of_the_boundary_day_and_ordered_oldest_first(tmp_path): - conn = _history_conn(tmp_path) - try: - for d in (10, 7, 3, 0): - _write(conn, _T0 - d * _DAY, level=100 + d) - week = cs.CensusStore.get_guild_history(conn, "Exordium", "Varsoon", 7, now=_T0) - assert [r["level"] for r in week] == [107, 103, 100] - assert [r["day"] for r in week] == sorted(r["day"] for r in week) - assert cs.CensusStore.get_guild_history(conn, "Exordium", "Varsoon", 400, now=_T0)[0]["level"] == 110 - finally: - conn.close() - - -def test_write_prunes_only_this_guilds_rows_past_retention(tmp_path): - conn = _history_conn(tmp_path) - try: - _write(conn, _T0 - 500 * _DAY, retention_days=400) - _write(conn, _T0 - 500 * _DAY, name="Other", retention_days=400) - _write(conn, _T0 - 100 * _DAY, retention_days=400) - _write(conn, _T0, retention_days=400) # prunes Exordium's 500-day-old row - mine = cs.CensusStore.get_guild_history(conn, "Exordium", "Varsoon", 1000, now=_T0) - assert len(mine) == 2 - assert cs.CensusStore.get_guild_history(conn, "Other", "Varsoon", 1000, now=_T0) # untouched - finally: - conn.close() - - -def test_history_missing_guild_is_empty(tmp_path): - conn = _history_conn(tmp_path) - try: - assert cs.CensusStore.get_guild_history(conn, "Nobody", "Varsoon", 90) == [] - finally: - conn.close() +def test_schema_has_guild_history(census_schema): + """Descendant of the SQLite init_db guild_history column check, against + the migration-owned schema.""" + with pg_conn(census_schema) as conn: + rows = conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'guild_history'", + (census_schema,), + ).fetchall() + cols = {r["column_name"] for r in rows} + assert { + "world", + "name_lower", + "day", + "captured_at", + "level", + "members", + "accounts", + "achievement_count", + "max_level_members", + "distinct_classes", + } == cols + + +def test_same_day_writes_collapse_to_one_row_last_capture_wins(conn): + _write(conn, _T0, level=300) + _write(conn, _T0 + 3600, level=301) # 15-minute refresh later the same UTC day + rows = cs.CensusStore.get_guild_history(conn, "exordium", "Varsoon", 7, now=_T0 + 3600) + assert len(rows) == 1 + assert rows[0]["level"] == 301 + assert rows[0]["captured_at"] == _T0 + 3600 + assert rows[0]["day"] == cs.CensusStore._utc_day(_T0) + assert rows[0]["members"] == 40 and rows[0]["distinct_classes"] == 9 + + +def test_window_is_inclusive_of_the_boundary_day_and_ordered_oldest_first(conn): + for d in (10, 7, 3, 0): + _write(conn, _T0 - d * _DAY, level=100 + d) + week = cs.CensusStore.get_guild_history(conn, "Exordium", "Varsoon", 7, now=_T0) + assert [r["level"] for r in week] == [107, 103, 100] + assert [r["day"] for r in week] == sorted(r["day"] for r in week) + assert cs.CensusStore.get_guild_history(conn, "Exordium", "Varsoon", 400, now=_T0)[0]["level"] == 110 + + +def test_write_prunes_only_this_guilds_rows_past_retention(conn): + _write(conn, _T0 - 500 * _DAY, retention_days=400) + _write(conn, _T0 - 500 * _DAY, name="Other", retention_days=400) + _write(conn, _T0 - 100 * _DAY, retention_days=400) + _write(conn, _T0, retention_days=400) # prunes Exordium's 500-day-old row + mine = cs.CensusStore.get_guild_history(conn, "Exordium", "Varsoon", 1000, now=_T0) + assert len(mine) == 2 + assert cs.CensusStore.get_guild_history(conn, "Other", "Varsoon", 1000, now=_T0) # untouched + + +def test_history_missing_guild_is_empty(conn): + assert cs.CensusStore.get_guild_history(conn, "Nobody", "Varsoon", 90) == [] diff --git a/tests/server/test_aa_census_store.py b/tests/server/test_aa_census_store.py index df15e065..5bd14bf8 100644 --- a/tests/server/test_aa_census_store.py +++ b/tests/server/test_aa_census_store.py @@ -5,22 +5,20 @@ - Warm store: census_store is served without calling Census. - Stale store record: census_store is served immediately, background refresh is spawned. - - init_db on a pre-existing census_store DB (without character_aas table) - doesn't crash — simulates an upgrade scenario. + - The migration-owned census schema carries the character_aas table + (descendant of the SQLite "init_db adds the table" upgrade test). """ from __future__ import annotations -import sqlite3 import time -from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import pytest from httpx import ASGITransport, AsyncClient from backend.census import store as cs -from backend.server.core.cache_keys import aa_cache_key +from tests.fixtures.pg import pg_conn # --------------------------------------------------------------------------- # Fixtures @@ -56,14 +54,12 @@ def _make_census_aas(name: str = "Testchar") -> MagicMock: @pytest.mark.asyncio -async def test_cold_cache_calls_census_and_persists(app, tmp_path): +async def test_cold_cache_calls_census_and_persists(app, census_schema): """On first fetch (no cache, no store), Census is queried and the result is written to census_store.""" - db_path = tmp_path / "backend.census.db" fake_aas = _make_census_aas("Coldchar") with ( - patch("backend.server.api.aa.census_store.path", db_path), patch("backend.server.core.census_lifecycle._clients", {}), patch("backend.server.core.census_lifecycle.CensusClient") as MockCC, ): @@ -80,7 +76,7 @@ async def test_cold_cache_calls_census_and_persists(app, tmp_path): assert data["total_spent"] == 5 # one node, tier=5 # Verify persisted to the store. - conn = cs.CensusStore(db_path).init_db() + conn = cs.CensusStore(census_schema).init_db() try: rec = cs.CensusStore.get_character_aas(conn, "Coldchar", "Varsoon") assert rec is not None @@ -95,11 +91,10 @@ async def test_cold_cache_calls_census_and_persists(app, tmp_path): @pytest.mark.asyncio -async def test_warm_store_skips_census(app, tmp_path): +async def test_warm_store_skips_census(app, census_schema): """When census_store has a fresh record, Census is NOT called.""" - db_path = tmp_path / "backend.census.db" # Pre-seed the store with a recent timestamp. - conn = cs.CensusStore(db_path).init_db() + conn = cs.CensusStore(census_schema).init_db() stored_data = { "character_name": "Stored", "total_spent": 10, @@ -110,7 +105,6 @@ async def test_warm_store_skips_census(app, tmp_path): conn.close() with ( - patch("backend.server.api.aa.census_store.path", db_path), patch("backend.server.api.aa.aa_cache") as mock_cache, patch("backend.server.core.census_lifecycle._clients", {}), patch("backend.server.core.census_lifecycle.CensusClient") as MockCC, @@ -139,14 +133,13 @@ async def test_warm_store_skips_census(app, tmp_path): @pytest.mark.asyncio -async def test_stale_store_returns_data_and_spawns_refresh(app, tmp_path): +async def test_stale_store_returns_data_and_spawns_refresh(app, census_schema): """A store record older than CHARACTER_STALE_S is served immediately but triggers a background refresh task.""" from backend.server.constants import CHARACTER_STALE_S - db_path = tmp_path / "backend.census.db" old_ts = int(time.time()) - CHARACTER_STALE_S - 60 # definitely stale - conn = cs.CensusStore(db_path).init_db() + conn = cs.CensusStore(census_schema).init_db() stored_data = { "character_name": "Stalechar", "total_spent": 7, @@ -160,11 +153,10 @@ async def test_stale_store_returns_data_and_spawns_refresh(app, tmp_path): def _fake_create_task(coro): tasks_created.append(coro) - # Don't actually run it. + coro.close() # never run it — just silence the un-awaited warning return MagicMock() with ( - patch("backend.server.api.aa.census_store.path", db_path), patch("backend.server.api.aa.aa_cache") as mock_cache, patch("backend.server.api.aa.asyncio.create_task", side_effect=_fake_create_task), patch("backend.server.core.census_lifecycle._clients", {}), @@ -183,28 +175,18 @@ def _fake_create_task(coro): # --------------------------------------------------------------------------- -# Migration safety: init_db on a pre-existing DB without character_aas table. +# Schema invariant: the migrations own the census DDL — the character_aas +# table must exist in any freshly-built census schema. Descendant of the +# SQLite "init_db on an old DB adds character_aas" upgrade test (no PG +# analog: store.init_db() creates nothing; db/migrations/0003_census.sql is +# the single source of schema truth). # --------------------------------------------------------------------------- -def test_init_db_on_old_schema_adds_character_aas_table(tmp_path): - """init_db on an existing census.db without character_aas should not crash - and should create the new table (simulates an upgrade scenario). - - Memory [test-migrations-against-old-db-shape]. - """ - db_path = tmp_path / "backend.census.db" - # Create the DB with only the original tables (no character_aas). - conn = sqlite3.connect(db_path) - conn.execute(cs._SQL["schema_characters"]) - conn.execute(cs._SQL["schema_guilds"]) - conn.commit() - conn.close() - - # Now run init_db — should add character_aas without raising. - conn = cs.CensusStore(db_path).init_db() - try: - tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} - assert "character_aas" in tables - finally: - conn.close() +def test_census_schema_has_character_aas_table(census_schema): + with pg_conn(census_schema) as conn: + rows = conn.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema = %s", + (census_schema,), + ).fetchall() + assert "character_aas" in {r["table_name"] for r in rows} diff --git a/tests/server/test_admin_parses.py b/tests/server/test_admin_parses.py index 3ea66eab..4c76cd8a 100644 --- a/tests/server/test_admin_parses.py +++ b/tests/server/test_admin_parses.py @@ -66,9 +66,7 @@ async def test_admin_parses_lists_including_hidden(app): "backend.server.api.admin.get_display_names_for_discord_ids", AsyncMock(return_value={"officer-7": "Vortex"}), ), - patch("backend.server.api.admin.parses_db.path") as mock_path, ): - mock_path.exists.return_value = True async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: r = await client.get("/api/admin/parses") assert r.status_code == 200 @@ -90,9 +88,7 @@ async def test_admin_parses_hidden_filter_reaches_the_store(app): patch("backend.server.api.admin._require_admin", _fake_admin), patch("backend.server.api.admin.parses_db.list_encounters_for_admin", list_mock), patch("backend.server.api.admin.parses_db.init_db", MagicMock(return_value=MagicMock())), - patch("backend.server.api.admin.parses_db.path") as mock_path, ): - mock_path.exists.return_value = True async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: r = await client.get("/api/admin/parses?hidden=true&search=exordium") assert r.status_code == 200 and r.json() == [] diff --git a/tests/server/test_admin_servers.py b/tests/server/test_admin_servers.py index 3d120961..d8e3f6ff 100644 --- a/tests/server/test_admin_servers.py +++ b/tests/server/test_admin_servers.py @@ -522,9 +522,7 @@ def _mock_admin_list(conn, *, search=None, limit=200, world=None, before=None, h patch("backend.server.api.admin.current_world", return_value="Wuoshi"), patch("backend.server.api.admin.parses_db.list_encounters_for_admin", _mock_admin_list), patch("backend.server.api.admin.parses_db.init_db", MagicMock(return_value=MagicMock())), - patch("backend.server.api.admin.parses_db.path") as mock_path, ): - mock_path.exists.return_value = True async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: r = await client.get("/api/admin/parses") diff --git a/tests/server/test_character.py b/tests/server/test_character.py index 6fd9ca93..a4f622ed 100644 --- a/tests/server/test_character.py +++ b/tests/server/test_character.py @@ -262,7 +262,7 @@ async def _fake_find(item_id, *args, **kwargs): @pytest.mark.asyncio -async def test_serve_path_self_heals_stored_placeholder(app, tmp_path, monkeypatch): +async def test_serve_path_self_heals_stored_placeholder(app, census_schema, monkeypatch): """End-to-end: stored character with 'Item #' in equipment → response carries the resolved name + tier + icon from items.db.""" import backend.server.api.character.views as charmodule @@ -281,8 +281,7 @@ async def test_serve_path_self_heals_stored_placeholder(app, tmp_path, monkeypat "adorn_slots": [], } ] - db_path = tmp_path / "backend.census.db" - conn = census_store.CensusStore(db_path).init_db() + conn = census_store.CensusStore(census_schema).init_db() try: census_store.CensusStore.upsert_character( conn, @@ -294,7 +293,6 @@ async def test_serve_path_self_heals_stored_placeholder(app, tmp_path, monkeypat ) finally: conn.close() - monkeypatch.setattr(census_store.store, "path", db_path) character_cache.delete(f"healme:{_WORLD.lower()}") async def _fake_find(item_id, *args, **kwargs): @@ -317,16 +315,15 @@ async def _fake_find(item_id, *args, **kwargs): @pytest.mark.asyncio -async def test_stored_data_served_without_census(app, tmp_path, monkeypatch): +async def test_stored_data_served_without_census(app, census_schema): """census_store hit + Census unreachable → 200 with stale=True, CensusClient never called.""" - import backend.server.api.character.views as charmodule from backend.census import store as census_store from backend.server.cache import character_cache from backend.server.config import WORLD as _WORLD - # Seed the census_store at an isolated tmp DB. - db_path = tmp_path / "backend.census.db" - conn = census_store.CensusStore(db_path).init_db() + # Seed the census_store in the isolated leased schema (census_schema has + # already re-pointed the shared store the endpoint reads through). + conn = census_store.CensusStore(census_schema).init_db() try: census_store.CensusStore.upsert_character( conn, @@ -339,9 +336,6 @@ async def test_stored_data_served_without_census(app, tmp_path, monkeypatch): finally: conn.close() - # Point the module at the tmp DB so the endpoint reads from it. - monkeypatch.setattr(census_store.store, "path", db_path) - # Ensure the in-memory cache is cold for this key. cache_key = f"stored:{_WORLD.lower()}" character_cache.delete(cache_key) @@ -359,7 +353,7 @@ async def test_stored_data_served_without_census(app, tmp_path, monkeypatch): @pytest.mark.asyncio -async def test_partial_roster_record_served_without_500(app, tmp_path, monkeypatch): +async def test_partial_roster_record_served_without_500(app, census_schema): """A partial store record (roster-sync: no id/world) must serve, not 500. Regression for the ValidationError seen in prod: CharacterResponse requires @@ -374,13 +368,11 @@ async def test_partial_roster_record_served_without_500(app, tmp_path, monkeypat # Roster-shaped partial blob: name/level/guild only — no id, no world. partial = {"name": "Verarec", "level": 90, "cls": "Wizard", "guild_name": "Test"} - db_path = tmp_path / "backend.census.db" - conn = census_store.CensusStore(db_path).init_db() + conn = census_store.CensusStore(census_schema).init_db() try: census_store.CensusStore.upsert_character(conn, "Verarec", _WORLD, partial, resolved=True, now=1000) finally: conn.close() - monkeypatch.setattr(census_store.store, "path", db_path) character_cache.delete(f"verarec:{_WORLD.lower()}") async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: diff --git a/tests/server/test_gear_sets.py b/tests/server/test_gear_sets.py index bfce82ca..c3098b3e 100644 --- a/tests/server/test_gear_sets.py +++ b/tests/server/test_gear_sets.py @@ -8,13 +8,12 @@ - Census down + nothing cached → 503. - Character with no saved sets → 200 with an empty list. - Unknown character → 404. - - init_db on a pre-existing census_store DB (without character_gear_sets - table) doesn't crash — simulates an upgrade scenario. + - The migration-owned census schema carries the character_gear_sets table + (descendant of the SQLite "init_db adds the table" upgrade test). """ from __future__ import annotations -import sqlite3 import time from unittest.mock import AsyncMock, MagicMock, patch @@ -23,6 +22,7 @@ from backend.census import store as cs from backend.census.models import AdornSlot, EquipmentSlot, GearSet +from tests.fixtures.pg import pg_conn # --------------------------------------------------------------------------- # Fixtures @@ -64,13 +64,10 @@ def _patch_char_id(char_id: str = "12345"): @pytest.mark.asyncio -async def test_cold_cache_calls_census_and_persists(app, tmp_path): +async def test_cold_cache_calls_census_and_persists(app, census_schema): """On first fetch (no cache, no store), Census is queried and the result is written to census_store.""" - db_path = tmp_path / "backend.census.db" - with ( - patch("backend.server.api.character.gear_sets.census_store.path", db_path), _patch_char_id("777"), patch("backend.server.core.census_lifecycle._clients", {}), patch("backend.server.core.census_lifecycle.CensusClient") as MockCC, @@ -91,7 +88,7 @@ async def test_cold_cache_calls_census_and_persists(app, tmp_path): mock_client.get_gear_sets.assert_awaited_once_with("777") # Verify persisted to the store. - conn = cs.CensusStore(db_path).init_db() + conn = cs.CensusStore(census_schema).init_db() try: rec = cs.CensusStore.get_character_gear_sets(conn, "Coldchar", "Varsoon") assert rec is not None @@ -106,16 +103,14 @@ async def test_cold_cache_calls_census_and_persists(app, tmp_path): @pytest.mark.asyncio -async def test_warm_store_skips_census(app, tmp_path): +async def test_warm_store_skips_census(app, census_schema): """When census_store has a fresh record, Census is NOT called.""" - db_path = tmp_path / "backend.census.db" - conn = cs.CensusStore(db_path).init_db() + conn = cs.CensusStore(census_schema).init_db() stored_data = {"character_name": "Stored", "sets": [{"name": "Old set", "ilvl": None, "equipment": []}]} cs.CensusStore.upsert_character_gear_sets(conn, "Stored", "Varsoon", stored_data, now=int(time.time())) conn.close() with ( - patch("backend.server.api.character.gear_sets.census_store.path", db_path), patch("backend.server.api.character.gear_sets.gear_sets_cache") as mock_cache, patch("backend.server.core.census_lifecycle._clients", {}), patch("backend.server.core.census_lifecycle.CensusClient") as MockCC, @@ -141,14 +136,13 @@ async def test_warm_store_skips_census(app, tmp_path): @pytest.mark.asyncio -async def test_stale_store_returns_data_and_spawns_refresh(app, tmp_path): +async def test_stale_store_returns_data_and_spawns_refresh(app, census_schema): """A store record older than CHARACTER_STALE_S is served immediately but triggers a background refresh task.""" from backend.server.constants import CHARACTER_STALE_S - db_path = tmp_path / "backend.census.db" old_ts = int(time.time()) - CHARACTER_STALE_S - 60 - conn = cs.CensusStore(db_path).init_db() + conn = cs.CensusStore(census_schema).init_db() stored_data = {"character_name": "Stalechar", "sets": []} cs.CensusStore.upsert_character_gear_sets(conn, "Stalechar", "Varsoon", stored_data, now=old_ts) conn.close() @@ -161,7 +155,6 @@ def _fake_create_task(coro): return MagicMock() with ( - patch("backend.server.api.character.gear_sets.census_store.path", db_path), patch("backend.server.api.character.gear_sets.gear_sets_cache") as mock_cache, patch("backend.server.api.character.gear_sets.asyncio.create_task", side_effect=_fake_create_task), patch("backend.server.core.census_lifecycle._clients", {}), @@ -183,13 +176,8 @@ def _fake_create_task(coro): @pytest.mark.asyncio -async def test_census_down_and_no_store_returns_503(app, tmp_path): - db_path = tmp_path / "backend.census.db" - - with ( - patch("backend.server.api.character.gear_sets.census_store.path", db_path), - patch("backend.server.census_health.is_down", return_value=True), - ): +async def test_census_down_and_no_store_returns_503(app, census_schema): + with patch("backend.server.census_health.is_down", return_value=True): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: resp = await client.get("/api/character/Nevercached/gear-sets") @@ -202,11 +190,8 @@ async def test_census_down_and_no_store_returns_503(app, tmp_path): @pytest.mark.asyncio -async def test_character_with_no_sets_returns_empty_list(app, tmp_path): - db_path = tmp_path / "backend.census.db" - +async def test_character_with_no_sets_returns_empty_list(app, census_schema): with ( - patch("backend.server.api.character.gear_sets.census_store.path", db_path), _patch_char_id(), patch("backend.server.core.census_lifecycle._clients", {}), patch("backend.server.core.census_lifecycle.CensusClient") as MockCC, @@ -222,7 +207,7 @@ async def test_character_with_no_sets_returns_empty_list(app, tmp_path): assert resp.json() == {"character_name": "Setless", "sets": []} # The empty answer is persisted too — repeat visitors skip Census. - conn = cs.CensusStore(db_path).init_db() + conn = cs.CensusStore(census_schema).init_db() try: assert cs.CensusStore.get_character_gear_sets(conn, "Setless", "Varsoon") is not None finally: @@ -235,13 +220,11 @@ async def test_character_with_no_sets_returns_empty_list(app, tmp_path): @pytest.mark.asyncio -async def test_unknown_character_returns_404(app, tmp_path): - db_path = tmp_path / "backend.census.db" +async def test_unknown_character_returns_404(app, census_schema): mock_char_cache = MagicMock() mock_char_cache.get_stale.return_value = (None, False) # no cached character with ( - patch("backend.server.api.character.gear_sets.census_store.path", db_path), patch("backend.server.api.character.gear_sets.character_cache", mock_char_cache), patch("backend.server.core.census_lifecycle._clients", {}), patch("backend.server.core.census_lifecycle.CensusClient") as MockCC, @@ -257,27 +240,18 @@ async def test_unknown_character_returns_404(app, tmp_path): # --------------------------------------------------------------------------- -# Migration safety: init_db on a pre-existing DB without character_gear_sets. +# Schema invariant: the migrations own the census DDL — the +# character_gear_sets table must exist in any freshly-built census schema. +# Descendant of the SQLite "init_db on an old DB adds character_gear_sets" +# upgrade test (no PG analog: store.init_db() creates nothing; +# db/migrations/0003_census.sql is the single source of schema truth). # --------------------------------------------------------------------------- -def test_init_db_on_old_schema_adds_gear_sets_table(tmp_path): - """init_db on an existing census.db without character_gear_sets should not - crash and should create the new table (simulates an upgrade scenario). - - Memory [test-migrations-against-old-db-shape]. - """ - db_path = tmp_path / "backend.census.db" - conn = sqlite3.connect(db_path) - conn.execute(cs._SQL["schema_characters"]) - conn.execute(cs._SQL["schema_guilds"]) - conn.execute(cs._SQL["schema_character_aas"]) - conn.commit() - conn.close() - - conn = cs.CensusStore(db_path).init_db() - try: - tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} - assert "character_gear_sets" in tables - finally: - conn.close() +def test_census_schema_has_gear_sets_table(census_schema): + with pg_conn(census_schema) as conn: + rows = conn.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema = %s", + (census_schema,), + ).fetchall() + assert "character_gear_sets" in {r["table_name"] for r in rows} diff --git a/tests/server/test_guild.py b/tests/server/test_guild.py index e61a531d..d81aa4ac 100644 --- a/tests/server/test_guild.py +++ b/tests/server/test_guild.py @@ -85,7 +85,6 @@ async def test_guild_roster_not_found(app): patch("backend.server.api.guild._persist_and_publish_guild", new_callable=AsyncMock) as mock_persist, ): mock_cache.get_stale.return_value = (None, False) - mock_cs.DB_PATH = census_store.DB_PATH mock_cs.init_db.return_value = MagicMock() mock_cs.get_guild.return_value = None # not in store mock_persist.return_value = None @@ -115,7 +114,6 @@ async def test_guild_roster_stale_triggers_background_refresh(app): patch("backend.server.census_refresh.request_guild_refresh", side_effect=lambda n: refresh_calls.append(n)), ): mock_cache.get_stale.return_value = (cached_roster, True) # stale in-memory - mock_cs.DB_PATH = census_store.DB_PATH mock_cs.init_db.return_value = MagicMock() # Store has data but it's old (last_resolved_at=1 → very stale) mock_cs.get_guild.return_value = {"data": stored_blob, "last_resolved_at": 1} @@ -163,7 +161,6 @@ async def test_guild_info_not_found(app): patch("backend.server.api.guild._persist_and_publish_guild", new_callable=AsyncMock) as mock_persist, ): mock_cache.get_stale.return_value = (None, False) - mock_cs.DB_PATH = census_store.DB_PATH mock_cs.init_db.return_value = MagicMock() mock_cs.get_guild.return_value = None mock_persist.return_value = None @@ -242,7 +239,6 @@ def _get_stale_side_effect(key): mock_cache.get_stale.side_effect = _get_stale_side_effect mock_cache.set.side_effect = _record_set # Store is empty for this test — exercise the live-fetch path - mock_cs.DB_PATH = census_store.DB_PATH mock_cs.init_db.return_value = MagicMock() mock_cs.get_guild.return_value = None @@ -261,11 +257,9 @@ def _get_stale_side_effect(key): @pytest.fixture -def tmp_census_db(tmp_path, monkeypatch): - """Seed a temporary census.db with one guild row and redirect DB_PATH to it.""" - db_path = tmp_path / "backend.census.db" - monkeypatch.setattr(census_store.store, "path", db_path) - monkeypatch.setenv("CENSUS_DB_PATH", str(db_path)) +def tmp_census_db(census_schema): + """Seed the leased census schema (shared store already re-pointed by + ``census_schema``) with one guild row; yields the schema name.""" roster_blob = GuildResponse( name="Exordium", world="Varsoon", @@ -294,10 +288,10 @@ def tmp_census_db(tmp_path, monkeypatch): type=0, ).model_dump() combined_blob = {"roster": roster_blob, "info": info_blob} - conn = census_store.CensusStore(db_path).init_db() + conn = census_store.CensusStore(census_schema).init_db() census_store.CensusStore.upsert_guild(conn, "Exordium", "Varsoon", combined_blob, now=1000) conn.close() - return db_path + return census_schema @pytest.mark.asyncio @@ -306,8 +300,6 @@ async def test_guild_roster_served_from_store_when_census_down(app, tmp_census_d import backend.server.census_health as _health_mod monkeypatch.setattr(_health_mod, "is_down", lambda: True) - # Redirect the module-level DB_PATH so the route's local init_db call uses the tmp db. - monkeypatch.setattr(census_store.store, "path", tmp_census_db) from backend.server.cache import guild_cache @@ -329,7 +321,6 @@ async def test_guild_info_served_from_store_when_census_down(app, tmp_census_db, import backend.server.census_health as _health_mod monkeypatch.setattr(_health_mod, "is_down", lambda: True) - monkeypatch.setattr(census_store.store, "path", tmp_census_db) from backend.server.cache import guild_cache @@ -352,7 +343,7 @@ async def test_guild_info_served_from_store_when_census_down(app, tmp_census_db, @pytest.mark.asyncio -async def test_persist_merges_offline_member_from_store(app, tmp_path, monkeypatch): +async def test_persist_merges_offline_member_from_store(app, census_schema): """_persist_and_publish_guild merges fresh-resolved members with last-good data for offline members (carried from the store), omits never-seen members, and does NOT bump the carried-forward member's last_resolved_at.""" @@ -361,12 +352,8 @@ async def test_persist_merges_offline_member_from_store(app, tmp_path, monkeypat from backend.server.api.guild import _persist_and_publish_guild from backend.server.cache import guild_cache - db_path = tmp_path / "backend.census.db" - monkeypatch.setattr(census_store.store, "path", db_path) - monkeypatch.setenv("CENSUS_DB_PATH", str(db_path)) - # Seed an offline alt that resolved in the PAST (now=1000). - conn = census_store.CensusStore(db_path).init_db() + conn = census_store.CensusStore(census_schema).init_db() census_store.CensusStore.upsert_character( conn, "OfflineAlt", @@ -442,7 +429,7 @@ async def test_persist_merges_offline_member_from_store(app, tmp_path, monkeypat ): await _persist_and_publish_guild("TestGuild") - conn = census_store.CensusStore(db_path).init_db() + conn = census_store.CensusStore(census_schema).init_db() try: guild_rec = census_store.CensusStore.get_guild(conn, "TestGuild", _WORLD) assert guild_rec is not None diff --git a/tests/server/test_guild_history.py b/tests/server/test_guild_history.py index 0776ad1f..017454d6 100644 --- a/tests/server/test_guild_history.py +++ b/tests/server/test_guild_history.py @@ -18,15 +18,15 @@ @pytest.fixture -def history_db(tmp_path, monkeypatch): - db_path = tmp_path / "backend.census.db" - monkeypatch.setattr(census_store.store, "path", db_path) +def history_db(census_schema): + """Leased scratch census schema (the shared store is already re-pointed + by ``census_schema``) with a cold guild-history cache key.""" guild_cache.delete(guild_history_key("Exordium", _WORLD)) - return db_path + return census_schema -def _seed(db_path, name: str, now: int, days_ago: list[int]) -> None: - conn = census_store.CensusStore(db_path).init_db() +def _seed(schema, name: str, now: int, days_ago: list[int]) -> None: + conn = census_store.CensusStore(schema).init_db() try: for d in days_ago: census_store.CensusStore.upsert_guild_history( diff --git a/tests/server/test_guild_recruitment.py b/tests/server/test_guild_recruitment.py index a0e00950..aa82d2c2 100644 --- a/tests/server/test_guild_recruitment.py +++ b/tests/server/test_guild_recruitment.py @@ -464,19 +464,20 @@ async def test_recruiting_list_is_world_scoped_with_tags_and_no_counts_without_c assert r.status_code == 200 body = r.json() assert [g["guild_name"] for g in body["guilds"]] == ["Alpha"] - assert body["guilds"][0]["member_count"] is None # no census.db in tests + assert body["guilds"][0]["member_count"] is None # no guild_history rows seeded assert body["available_tags"] == list(RECRUITMENT_TAGS) -async def test_recruiting_list_joins_member_counts_from_guild_history(app, tmp_path, monkeypatch): +async def test_recruiting_list_joins_member_counts_from_guild_history(app, census_schema): from backend.census.store import CensusStore - cs = CensusStore(tmp_path / "census.db") + # census_schema re-points the shared store (the instance recruitment_api + # imported) at the leased schema; seed it through a scoped conn. + cs = CensusStore(census_schema) conn = cs.init_db() cs.upsert_guild_history(conn, "Alpha", _WORLD, {"members": 55}, now=1_800_000_000, retention_days=400) cs.upsert_guild_history(conn, "Alpha", _WORLD, {"members": 61}, now=1_800_100_000, retention_days=400) conn.close() - monkeypatch.setattr(recruitment_api, "census_store", cs) await _save_profile(gid=1, name="Alpha") await _save_profile(gid=2, name="NoHistory") async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: diff --git a/tests/server/test_parses_ingest_cache.py b/tests/server/test_parses_ingest_cache.py index f2d456b4..2991c5ed 100644 --- a/tests/server/test_parses_ingest_cache.py +++ b/tests/server/test_parses_ingest_cache.py @@ -27,11 +27,10 @@ def model_dump(self): @pytest.fixture -def store_db(tmp_path, monkeypatch): - """Temp census_store wired into the ingest path via DB_PATH.""" - db = tmp_path / "census.db" - monkeypatch.setattr(census_store.store, "path", db) - conn = census_store.CensusStore(db).init_db() +def store_db(census_schema): + """Leased census schema wired into the ingest path (the ``census_schema`` + fixture re-points the shared store); yields a seeding/assertion conn.""" + conn = census_store.CensusStore(census_schema).init_db() yield conn conn.close() diff --git a/tests/server/test_parses_top_n.py b/tests/server/test_parses_top_n.py index 31e00542..be74a500 100644 --- a/tests/server/test_parses_top_n.py +++ b/tests/server/test_parses_top_n.py @@ -8,7 +8,7 @@ from __future__ import annotations -import sqlite3 +from typing import Any import pytest @@ -16,12 +16,12 @@ @pytest.fixture -def conn() -> sqlite3.Connection: - """In-memory sqlite with just the columns the helpers read. +def conn(parses_db_conn: Any) -> Any: + """Scratch parses-schema connection (dict rows, ``%s`` params). - Schema is deliberately minimal — we don't want the helpers' behaviour - to depend on any column we don't actually query, and the parses DB - schema itself is tested elsewhere. + Postgres enforces the combatants → encounters FK, so ``_insert`` + lazily creates a stub encounter row per encounter_id — the helpers + under test still only read the combatant columns. Phase 4 (2026-05-30) switched the top-N helpers from the legacy multi-word/Unknown predicate to ``is_player = 1``. The test @@ -30,32 +30,26 @@ def conn() -> sqlite3.Connection: so the test semantics ("regex/multi-word names don't count") are preserved without changing every test body. """ - c = sqlite3.connect(":memory:") - c.execute( - """ - CREATE TABLE combatants ( - id INTEGER PRIMARY KEY, - encounter_id INTEGER NOT NULL, - name TEXT NOT NULL, - ally INTEGER NOT NULL, - encdps REAL NOT NULL DEFAULT 0, - is_player INTEGER - ) - """ - ) - return c + return parses_db_conn -def _insert(conn: sqlite3.Connection, **kwargs) -> None: +def _insert(conn: Any, **kwargs) -> None: # Derive is_player from the legacy predicate so existing test # bodies that exercised the multi-word/Unknown filter still pass # under the new is_player-based SQL helpers. name = kwargs["name"] ally = kwargs["ally"] + encounter_id = kwargs["encounter_id"] is_player = 1 if (ally == 1 and name and name != "Unknown" and " " not in name) else 0 conn.execute( - "INSERT INTO combatants (encounter_id, name, ally, encdps, is_player) VALUES (?, ?, ?, ?, ?)", - (kwargs["encounter_id"], name, ally, kwargs["encdps"], is_player), + "INSERT INTO encounters (id, act_encid, title, started_at, ended_at, duration_s, source_dsn, ingested_at) " + "OVERRIDING SYSTEM VALUE VALUES (%s, %s, 'Fight', 0, 0, 0, 'test', 0) " + "ON CONFLICT (id) DO NOTHING", + (encounter_id, f"enc-{encounter_id}"), + ) + conn.execute( + "INSERT INTO combatants (encounter_id, name, ally, encdps, is_player) VALUES (%s, %s, %s, %s, %s)", + (encounter_id, name, ally, kwargs["encdps"], is_player), ) From ffe0dfe15c95d95472431e1293b8d3758997e067 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 17:04:15 +0100 Subject: [PATCH 23/32] feat(pg): lockdown migration + post-cutover deploy config + nightly pg_dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0006_lockdown (-- global): RLS enabled on every family table + guarded revokes from Supabase's anon/authenticated — the auto REST API can't read the data even though public stays the only exposed (and empty) schema. railway.toml startCommand drops litestream entirely (deleted litestream.yml): mkdir only the read-only mirror dirs, exec main.py; lifespan migrations make deploy = migrate. .env.example: DATABASE_URL chain + TEST_DATABASE_URL documented; retired DB path vars removed. New nightly pg-backup workflow: pg_dump -Fc of the five schemas to the repurposed R2 bucket with 30-day in-action pruning. Co-Authored-By: Claude Fable 5 --- .env.example | 32 +++++------- .github/workflows/pg-backup.yml | 58 +++++++++++++++++++++ db/migrations/0006_lockdown.sql | 33 ++++++++++++ litestream.yml | 91 --------------------------------- railway.toml | 28 +++++----- 5 files changed, 117 insertions(+), 125 deletions(-) create mode 100644 .github/workflows/pg-backup.yml create mode 100644 db/migrations/0006_lockdown.sql delete mode 100644 litestream.yml diff --git a/.env.example b/.env.example index 7e81b93d..9e35c58d 100644 --- a/.env.example +++ b/.env.example @@ -133,18 +133,21 @@ HTTPS_ONLY=true # LOG_FORMAT: text (default, human-readable) or json (Railway / structured aggregators) # LOG_FORMAT=text -# ── Database Paths ──────────────────────────────────────────────────────────── -# Every DB has a DB__PATH override. Defaults sit under data/ alongside +# ── Postgres (writable families: users / parses / census / zones / raids) ──── +# One database, five schemas (db/migrations/). The app resolves the DSN as +# DATABASE_URL → SUPABASE_DB_URL → POSTGRES_CONNECTION_STRING. Production +# uses the Supabase SESSION pooler (:5432, sslmode=require) — NOT the +# transaction pooler (:6543), which breaks prepared statements and the +# per-checkout search_path. Migrations apply at app startup. +# DATABASE_URL=postgresql://postgres.:@.pooler.supabase.com:5432/postgres?sslmode=require +# +# Local test suite target (tests/fixtures/pg.py; created automatically): +# TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/eq2lexicon_test + +# ── SQLite catalogue paths (read-only Census mirrors) ──────────────────────── +# The remaining file-backed catalogues. Defaults sit under data/ alongside # the application; in Railway, point each one at the persistent volume mount. -# The DB_ prefix keeps them grouped alphabetically in your env-var list. # -# DB_USERS_PATH Discord logins, claims, api_tokens, roles, notifications. -# Backed up via litestream → R2. -# DB_PARSES_PATH Normalised ACT parse uploads (the rankings + DPS boards). -# Backed up via litestream → R2. -# DB_CENSUS_PATH Persistent best-known cache of Census character/guild -# lookups (web read path never blocks on the Census API). -# Lives on the volume so it survives redeploys. # DB_ITEMS_PATH Census item cache (item tooltips, stats, set bonuses). # Local-generated via scripts/download_items.py. # DB_SPELLS_PATH Census spell cache (icons, tiers, given_by). Local-only. @@ -155,20 +158,11 @@ HTTPS_ONLY=true # DB_AAS_PATH AA tree + node catalogue built from data/AAs/trees via # scripts/build_aas_db.py. Committed pre-populated (like # classes.db) so no upload is needed. -# DB_ZONES_PATH Zone + raid-boss roster reference. Built locally from -# scripts/dev/eq2_raid_bosses.review.txt — manual upload. -# DB_RAIDS_PATH Raid strategies + ACT trigger packs (human-edited content). -# Manual upload — accumulates edits, do not clobber. # DB_ACT_EXPORT_PATH Path to ACT's OWN export.db that parses/act_reader.py reads # from at ingest time (dev-only; not used in production). # -# DB_USERS_PATH=/data/users.db -# DB_PARSES_PATH=/data/parses/parses.db -# DB_CENSUS_PATH=/data/census/census.db # DB_ITEMS_PATH=/data/items/items.db # DB_SPELLS_PATH=/data/spells/spells.db # DB_RECIPES_PATH=/data/recipes/recipes.db # DB_AAS_PATH=/data/AAs/aas.db -# DB_ZONES_PATH=/data/zones/zones.db -# DB_RAIDS_PATH=/data/raids/raids.db # DB_ACT_EXPORT_PATH=/data/parses/act_export.db diff --git a/.github/workflows/pg-backup.yml b/.github/workflows/pg-backup.yml new file mode 100644 index 00000000..8e001928 --- /dev/null +++ b/.github/workflows/pg-backup.yml @@ -0,0 +1,58 @@ +name: Nightly Postgres backup + +# pg_dump of the five family schemas → Cloudflare R2 (the bucket litestream +# used pre-migration, repurposed). Restore drill: +# pg_restore -d "$DATABASE_URL" --clean --if-exists eq2lexicon-.dump +# Secrets: SUPABASE_DB_URL (session pooler, sslmode=require), R2_ENDPOINT, +# R2_BUCKET, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY. + +on: + schedule: + - cron: "17 4 * * *" # 04:17 UTC daily + workflow_dispatch: + +jobs: + dump: + runs-on: ubuntu-latest + steps: + - name: Install PostgreSQL 17 client + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc > /dev/null + sudo apt-get update -qq + sudo apt-get install -y -qq postgresql-client-17 + + - name: pg_dump (family schemas only) + env: + DATABASE_URL: ${{ secrets.SUPABASE_DB_URL }} + run: | + ts=$(date -u +%Y%m%dT%H%M%SZ) + dump="eq2lexicon-$ts.dump" + # Only our schemas — Supabase-internal schemas (auth/storage/…) + # are not ours to dump and would fail on permissions. + pg_dump -Fc -d "$DATABASE_URL" \ + -n users -n parses -n census -n zones -n raids -n public \ + -f "$dump" + ls -la "$dump" + echo "DUMP=$dump" >> "$GITHUB_ENV" + + - name: Upload to R2 and prune dumps older than 30 days + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} + R2_BUCKET: ${{ secrets.R2_BUCKET }} + run: | + aws s3 cp "$DUMP" "s3://$R2_BUCKET/pgdump/$DUMP" --endpoint-url "$R2_ENDPOINT" + cutoff=$(date -u -d '30 days ago' +%Y%m%dT%H%M%SZ) + aws s3 ls "s3://$R2_BUCKET/pgdump/" --endpoint-url "$R2_ENDPOINT" \ + | awk '{print $4}' \ + | while read -r f; do + [ -n "$f" ] || continue + tspart=$(printf '%s' "$f" | sed -E 's/^eq2lexicon-([0-9TZ]+)\.dump$/\1/') + if [ "$tspart" != "$f" ] && [ "$tspart" \< "$cutoff" ]; then + echo "pruning $f" + aws s3 rm "s3://$R2_BUCKET/pgdump/$f" --endpoint-url "$R2_ENDPOINT" + fi + done diff --git a/db/migrations/0006_lockdown.sql b/db/migrations/0006_lockdown.sql new file mode 100644 index 00000000..35c9a89d --- /dev/null +++ b/db/migrations/0006_lockdown.sql @@ -0,0 +1,33 @@ +-- global +-- Supabase lockdown (belt and braces). The app connects as the schema +-- OWNER, which bypasses RLS — enabling RLS on every family table plus +-- revoking the PostgREST roles means the auto-generated API can't read +-- the data even if a schema were ever added to the exposed list +-- (`public` stays the only exposed schema, and it is empty). +-- +-- NOTE: this runs once (migration ledger). A future migration that adds +-- a table must ENABLE ROW LEVEL SECURITY on it itself. + +DO $$ +DECLARE r record; +BEGIN + FOR r IN SELECT schemaname, tablename FROM pg_tables + WHERE schemaname IN ('users', 'parses', 'census', 'zones', 'raids') + LOOP + EXECUTE format('ALTER TABLE %I.%I ENABLE ROW LEVEL SECURITY', r.schemaname, r.tablename); + END LOOP; +END $$; + +-- Supabase ships anon / authenticated; plain Postgres (local dev, tests, +-- CI) doesn't — guard every revoke so the migration runs everywhere. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'anon') THEN + REVOKE ALL ON ALL TABLES IN SCHEMA users, parses, census, zones, raids FROM anon; + REVOKE USAGE ON SCHEMA users, parses, census, zones, raids FROM anon; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticated') THEN + REVOKE ALL ON ALL TABLES IN SCHEMA users, parses, census, zones, raids FROM authenticated; + REVOKE USAGE ON SCHEMA users, parses, census, zones, raids FROM authenticated; + END IF; +END $$; diff --git a/litestream.yml b/litestream.yml deleted file mode 100644 index 36644947..00000000 --- a/litestream.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Litestream — continuous replication of the SQLite DBs to Cloudflare R2. -# -# Four databases are backed up: the parses corpus (parses.db), the -# Discord/claims/api_tokens store (users.db), the zone/boss-roster -# catalogue (zones.db), and the raid strategy/timers store -# (raids.db — strategy markdown is hand-edited and revision-tracked, so -# losing it on a volume wipe would erase audit history). -# -# PATHS MUST MATCH THE APP'S DB_*_PATH RESOLUTION. The app reads -# DB_USERS_PATH / DB_PARSES_PATH / DB_ZONES_PATH / DB_RAIDS_PATH at -# import time (backend/eq2db/raids.py + backend/server/parses/db.py + -# backend/server/db/) and uses -# those for every read and write. This file MUST reference the same -# files via the same env vars — otherwise litestream watches one path -# while the app writes to another, the configured-but-nonexistent file -# never appears, and the R2 bucket stays empty. -# -# Configuration via env vars (set in Railway → Variables): -# DB_USERS_PATH /data/users.db (or wherever the volume is mounted) -# DB_PARSES_PATH /data/parses.db -# DB_ZONES_PATH /data/zones.db -# DB_RAIDS_PATH /data/raids.db -# R2_ENDPOINT https://.r2.cloudflarestorage.com -# R2_BUCKET e.g. eq2lexicon-backups -# R2_ACCESS_KEY_ID R2 API token "Access Key ID" -# R2_SECRET_ACCESS_KEY R2 API token "Secret Access Key" -# -# Recovery: on container start, railway.toml's startCommand first runs -# `litestream restore -if-replica-exists` for each DB, so a wiped volume -# auto-reseeds from R2 before the app starts. After that the long-running -# `litestream replicate` keeps all four DBs in sync (RPO ~10 s). - -dbs: - # NOTE on env-var syntax: litestream uses Go's os.ExpandEnv, which only - # supports `${VAR}` — NOT `${VAR:-default}`. Bash-style defaults parse as - # a single (nonexistent) env-var name and substitute to empty string, - # which leaves the path empty and litestream then tries to open `/app` - # (the cwd) as a database. So: no defaults here. The env vars MUST be - # set on the Railway service (see CLAUDE.md → Environment variables). - - path: ${DB_PARSES_PATH} - replicas: - - type: s3 - endpoint: ${R2_ENDPOINT} - bucket: ${R2_BUCKET} - path: parses - access-key-id: ${R2_ACCESS_KEY_ID} - secret-access-key: ${R2_SECRET_ACCESS_KEY} - region: auto - # Keep ~7 days of recoverable history; older snapshots GC'd. - retention: 168h - retention-check-interval: 24h - snapshot-interval: 24h - - - path: ${DB_USERS_PATH} - replicas: - - type: s3 - endpoint: ${R2_ENDPOINT} - bucket: ${R2_BUCKET} - path: users - access-key-id: ${R2_ACCESS_KEY_ID} - secret-access-key: ${R2_SECRET_ACCESS_KEY} - region: auto - retention: 168h - retention-check-interval: 24h - snapshot-interval: 24h - - - path: ${DB_ZONES_PATH} - replicas: - - type: s3 - endpoint: ${R2_ENDPOINT} - bucket: ${R2_BUCKET} - path: zones - access-key-id: ${R2_ACCESS_KEY_ID} - secret-access-key: ${R2_SECRET_ACCESS_KEY} - region: auto - retention: 168h - retention-check-interval: 24h - snapshot-interval: 24h - - - path: ${DB_RAIDS_PATH} - replicas: - - type: s3 - endpoint: ${R2_ENDPOINT} - bucket: ${R2_BUCKET} - path: raids - access-key-id: ${R2_ACCESS_KEY_ID} - secret-access-key: ${R2_SECRET_ACCESS_KEY} - region: auto - retention: 168h - retention-check-interval: 24h - snapshot-interval: 24h diff --git a/railway.toml b/railway.toml index d5b9c8c3..1bbbd2b7 100644 --- a/railway.toml +++ b/railway.toml @@ -2,15 +2,12 @@ builder = "nixpacks" [deploy] -# Start sequence (see litestream.yml for context): -# 1. Make sure the volume dirs exist (first deploy after volume creation). -# 2. Restore both DBs from R2 if-replica-exists (no-op if the volume -# already has them, recovers them if the volume was wiped, and skipped -# cleanly when the R2 env vars aren't set yet — `|| true`). -# 3. Exec the app under litestream's supervision so every write is -# streamed to R2 in near-real-time. -# The trailing `exec` matters: litestream then becomes PID 1's child and -# Railway's restart policy applies to the app process correctly. +# Start sequence (post-Postgres-migration): +# 1. Make sure the read-only catalogue mirror dirs exist on the volume +# (items/spells/recipes are still hand-copied SQLite files; the +# writable families live in Supabase Postgres — DATABASE_URL). +# 2. Exec the app. Schema migrations run inside the app lifespan +# (backend/pg_migrate.py) — a deploy IS a migration run. # # Python binary discovery (2026-05-27): we MUST use the project's venv # (`/app/.venv/bin/python`) — that's where `uv sync` installed the @@ -24,16 +21,17 @@ builder = "nixpacks" # dump of every candidate runs unconditionally so any future Nixpacks # layout change is visible in the first 10 log lines without another # round-trip through build logs. -startCommand = "set -x; USERS_DB=${DB_USERS_PATH:-/app/data/users.db}; PARSES_DB=${DB_PARSES_PATH:-/app/data/parses/parses.db}; ZONES_DB=${DB_ZONES_PATH:-/app/data/zones/zones.db}; RAIDS_DB=${DB_RAIDS_PATH:-/app/data/raids/raids.db}; mkdir -p \"$(dirname \"$USERS_DB\")\" \"$(dirname \"$PARSES_DB\")\" \"$(dirname \"$ZONES_DB\")\" \"$(dirname \"$RAIDS_DB\")\"; echo '=== python path discovery ==='; (which python python3 2>&1 || true); (ls -la /app/.venv/bin/ 2>&1 | head -20 || true); (ls -la /opt/venv/bin/ 2>&1 | head -20 || true); echo \"PATH=$PATH\"; echo '=== end discovery ==='; PYTHON_BIN=''; for c in /app/.venv/bin/python /opt/venv/bin/python; do if [ -x \"$c\" ]; then PYTHON_BIN=\"$c\"; break; fi; done; if [ -z \"$PYTHON_BIN\" ]; then PYTHON_BIN=$(command -v python || command -v python3 || echo ''); fi; if [ -z \"$PYTHON_BIN\" ]; then echo 'FATAL: no python found in project venv or on PATH' >&2; exit 78; fi; echo \"Resolved Python: $PYTHON_BIN\"; echo \"Resolved DB paths for litestream: USERS=$USERS_DB PARSES=$PARSES_DB ZONES=$ZONES_DB RAIDS=$RAIDS_DB\"; (litestream restore -if-replica-exists -config /app/litestream.yml \"$USERS_DB\" || true); (litestream restore -if-replica-exists -config /app/litestream.yml \"$PARSES_DB\" || true); (litestream restore -if-replica-exists -config /app/litestream.yml \"$ZONES_DB\" || true); (litestream restore -if-replica-exists -config /app/litestream.yml \"$RAIDS_DB\" || true); exec litestream replicate -config /app/litestream.yml -exec \"$PYTHON_BIN main.py\"" +startCommand = "set -x; ITEMS_DB=${DB_ITEMS_PATH:-/app/data/items/items.db}; SPELLS_DB=${DB_SPELLS_PATH:-/app/data/spells/spells.db}; RECIPES_DB=${DB_RECIPES_PATH:-/app/data/recipes/recipes.db}; mkdir -p \"$(dirname \"$ITEMS_DB\")\" \"$(dirname \"$SPELLS_DB\")\" \"$(dirname \"$RECIPES_DB\")\"; echo '=== python path discovery ==='; (which python python3 2>&1 || true); (ls -la /app/.venv/bin/ 2>&1 | head -20 || true); (ls -la /opt/venv/bin/ 2>&1 | head -20 || true); echo \"PATH=$PATH\"; echo '=== end discovery ==='; PYTHON_BIN=''; for c in /app/.venv/bin/python /opt/venv/bin/python; do if [ -x \"$c\" ]; then PYTHON_BIN=\"$c\"; break; fi; done; if [ -z \"$PYTHON_BIN\" ]; then PYTHON_BIN=$(command -v python || command -v python3 || echo ''); fi; if [ -z \"$PYTHON_BIN\" ]; then echo 'FATAL: no python found in project venv or on PATH' >&2; exit 78; fi; echo \"Resolved Python: $PYTHON_BIN\"; exec \"$PYTHON_BIN\" main.py" restartPolicyType = "ON_FAILURE" restartPolicyMaxRetries = 10 # Don't route traffic (or call the deploy live) until the app actually # serves — without this Railway switches the edge to the new container the -# moment it starts, while litestream restore + python imports + lifespan -# are still running, and every request 502s through Cloudflare. NOTE: with -# an attached volume Railway still deploys WITH downtime (the volume can -# only mount one container), so the goal is a window of a few seconds: -# fast old-process exit (see main.py signal handling) + gated cutover here. +# moment it starts, while python imports + lifespan (pool open + pending +# migrations) are still running, and every request 502s through +# Cloudflare. NOTE: with an attached volume Railway still deploys WITH +# downtime (the volume can only mount one container), so the goal is a +# window of a few seconds: fast old-process exit (see main.py signal +# handling) + gated cutover here. healthcheckPath = "/api/health" healthcheckTimeout = 300 watchPaths = ["**", "!ops/**", "!docs/**"] From dc4c51f284afad46fc1930790b6930670c15d386 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 17:33:21 +0100 Subject: [PATCH 24/32] =?UTF-8?q?feat(pg):=20zones=20+=20raids=20families?= =?UTF-8?q?=20on=20psycopg=20=E2=80=94=20catalogues,=20routes,=20scripts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ZoneCatalogue/RaidCatalogue on PgCatalogue (schemas zones/raids); the act/* and raid_strategies routes, rankings' zones reads, api/zones mob-title matching (= ANY replaces the chunked IN loop), metrics raids counts (the _DBCollector sqlite conn-cache is gone — nothing used it), and the dev scripts all ride the pooled proxy. build_zones_db.py now upserts zones by stable id and prunes removed names with a printed cascade warning, so metadata refreshes become a script run against DATABASE_URL and curator boss rosters survive (the manual-volume-upload workflow dies). NULLS FIRST/LAST pinned where SQLite and PG disagree (featured lanes, expansion_year sorts). 0004 fix: featured_raid_zones position/category (post-launch SQLite ALTERs my translation missed — caught by the conversion agent). app.py startup parses checkout now closes (leaked one sync-pool slot). SchemaBound gains the no-op clear_caches for PathBound parity. Verified by zones/raids live smokes, repo-wide pyright 0 errors, and the app lifespan boot. Co-Authored-By: Claude Fable 5 --- backend/db_catalogue.py | 5 + backend/eq2db/raids.py | 229 +++---- backend/eq2db/raids.sql | 266 ++------ backend/eq2db/zones.py | 836 ++++++++++++------------- backend/eq2db/zones.sql | 375 ++++------- backend/server/api/act/_shared.py | 54 +- backend/server/api/act/categories.py | 48 +- backend/server/api/act/pack.py | 6 +- backend/server/api/act/spell_timers.py | 7 +- backend/server/api/act/triggers.py | 18 +- backend/server/api/raid_strategies.py | 43 +- backend/server/api/raid_strategies.sql | 30 +- backend/server/api/rankings.py | 88 +-- backend/server/api/rankings.sql | 15 +- backend/server/api/zones.py | 36 +- backend/server/api/zones.sql | 17 +- backend/server/app.py | 16 +- backend/server/metrics.py | 93 +-- backend/server/metrics.sql | 6 +- db/migrations/0004_zones.sql | 9 +- scripts/build_zones_db.py | 48 +- scripts/dev/_smoke_test_zones_db.py | 87 ++- scripts/dev/apply_polish.py | 16 +- scripts/dev/apply_rebalance.py | 28 +- scripts/dev/apply_zone_overviews.py | 16 +- scripts/dev/export_for_polish.py | 9 +- scripts/dev/export_for_rebalance.py | 6 +- scripts/dev/export_zone_overviews.py | 12 +- scripts/dev/ingest_act_triggers.py | 33 +- scripts/dev/seed_fake_parses.py | 27 +- tests/conftest.py | 8 +- 31 files changed, 984 insertions(+), 1503 deletions(-) diff --git a/backend/db_catalogue.py b/backend/db_catalogue.py index 3c1c2243..7253e0c7 100644 --- a/backend/db_catalogue.py +++ b/backend/db_catalogue.py @@ -344,6 +344,11 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash((type(self), self.schema)) + def clear_caches(self) -> None: + """Reset per-instance caches — the uniform catalogue surface + (mirrors :meth:`PathBound.clear_caches`). Default: no caches; + subclasses holding caches override.""" + class PgStoreBase(SchemaBound): """Base for the async (formerly aiosqlite) users-family domain stores. diff --git a/backend/eq2db/raids.py b/backend/eq2db/raids.py index bc8cdd0f..020fadf1 100644 --- a/backend/eq2db/raids.py +++ b/backend/eq2db/raids.py @@ -1,8 +1,8 @@ """ -Local SQLite catalogue of EverQuest 2 raid strategies. +Postgres catalogue of EverQuest 2 raid strategies (the ``raids`` schema). -Companion to ``backend/eq2db/zones.py`` — zones.db is read-only reference -data rebuilt from JSON; this DB accumulates strategy content sourced +Companion to ``backend/eq2db/zones.py`` — zones is read-only reference +data rebuilt from JSON; this family accumulates strategy content sourced initially from the EQ2 wiki (EQ2i / Fandom) and then progressively hand-edited by guild officers. @@ -10,12 +10,12 @@ align with the TLE-server content cycle. Live-expansion strategies are out of scope for the moment. -Schema (six tables): +Tables (schema DDL in db/migrations/0005_raids.sql): * **raid_zones** — one row per raid zone with zone-level metadata (access, level range, etc.). - Loose FK by ``zone_name`` to zones.db - (different DB file — no enforced FK). + Loose FK by ``zone_name`` to the zones + family (different schema — no enforced FK). * **raid_encounters** — one row per named boss within a raid zone. ``strategy_md`` is a single markdown blob (PoC simplicity; can @@ -25,47 +25,45 @@ raid_encounters.strategy_md writes a row here with before/after + editor identity + timestamp. + * **raid_zone_revisions** — the zone-overview counterpart. * **act_triggers** / **act_spell_timers** — per-encounter ACT trigger and spell-timer rows (the /act editor). - * **_meta** — provenance: built_at, scraper_source, - source_count, etc. The `source` column on raid_zones / raid_encounters tracks where the content came from: * 'eq2i_scrape' — auto-extracted from the wiki, untouched - * 'manual' — added or edited by a human via the future editor + * 'manual' — added or edited by a human via the editor * 'parse_data' — derived from encounter parses (e.g. mechanic timing confirmed from log analysis; future feature) A row can transition: 'eq2i_scrape' → 'manual' on first hand-edit. The revision history preserves the original scrape for audit. -All behaviour lives on :class:`RaidCatalogue` (the eq2db data-interface -convention — see AACatalogue / SpellCatalogue): path-based reads are -instance methods; the conn-taking write helpers (upserts, mirrors, ACT -trigger/timer writes) are staticmethods on the same class so consumers -import ONE name — the shared ``catalogue`` instance. The former -``raids_act`` re-export layer is folded into the class. +All behaviour lives on :class:`RaidCatalogue` (the catalogue convention — +see backend/db_catalogue.py): the shared module-level ``catalogue`` +instance is the runtime entry point (consumers alias it ``raids_db``); +``init_db()`` returns a pooled schema-scoped connection proxy. The +conn-taking write helpers (upserts, mirrors, ACT trigger/timer writes) +are staticmethods — callers batch several writes per connection; the +per-call read helpers open (and release) their own pooled checkout. """ from __future__ import annotations -import sqlite3 +import logging import time -from pathlib import Path +from collections.abc import Sequence +from typing import Any -from backend.db_catalogue import BaseCatalogue -from backend.db_helpers import resolve_db_path +from backend.db_catalogue import PgCatalogue from backend.sql_loader import load_sql -_SQL = load_sql(__file__) - -# --------------------------------------------------------------------------- -# Config -# --------------------------------------------------------------------------- +_log = logging.getLogger(__name__) +_SQL = load_sql(__file__) -DB_PATH: Path = resolve_db_path("DB_RAIDS_PATH", "raids", "raids.db") +#: Postgres schema the raids family lives in (db/migrations/0005_raids.sql). +SCHEMA = "raids" # Source provenance tokens for the `source` columns. Centralised so @@ -77,11 +75,6 @@ VALID_SOURCES: frozenset[str] = frozenset({SOURCE_SCRAPE, SOURCE_MANUAL, SOURCE_PARSE}) -# Schema (CREATE TABLE / INDEX) lives in raids.sql; init_db runs each block. - -# `_meta` get/set is shared across every eq2db module — see backend/eq2db/_meta.py. -from backend.eq2db._meta import get_meta, set_meta # noqa: E402,F401 - # --------------------------------------------------------------------------- # Column lists # --------------------------------------------------------------------------- @@ -104,24 +97,16 @@ "last_edited_at, last_edited_by, created_at" ) -# EQ2Parser-enrichment columns added 2026-08 — ALTERs bring a pre-existing -# raids.db up to the CREATE TABLE shape above (idempotent via -# _apply_migrations' duplicate-column skip). -_ACT_ENRICHMENT_MIGRATIONS = ( - "ALTER TABLE act_triggers ADD COLUMN cooldown_seconds REAL NOT NULL DEFAULT 1.0", - "ALTER TABLE act_spell_timers ADD COLUMN damage_type TEXT NOT NULL DEFAULT ''", - "ALTER TABLE act_spell_timers ADD COLUMN control_effect TEXT NOT NULL DEFAULT ''", -) +class RaidCatalogue(PgCatalogue): + """Read/write access to the raids schema. -class RaidCatalogue(BaseCatalogue): - """Read (and build) access to one raids.db file. - - The eq2db data-interface convention (see AACatalogue / SpellCatalogue): - the DB path lives on the instance; the shared module-level ``catalogue`` - is the runtime entry point, and tests construct ``RaidCatalogue(tmp_db)``. - Path-based reads are instance methods; write helpers take an open conn - (callers batch several writes per transaction) and are staticmethods. + The catalogue convention (see AACatalogue / SpellCatalogue / the other + Postgres families): the shared module-level ``catalogue`` is the runtime + entry point, and tests construct ``RaidCatalogue(scratch_schema)``. + Write helpers take an open conn (callers batch several writes per + transaction) and are staticmethods; the read helpers open a pooled + checkout per call (mirroring the old per-call sqlite connections). The provenance tokens are mirrored as class attributes so consumers holding the catalogue can write ``raids_db.SOURCE_MANUAL``. @@ -132,47 +117,26 @@ class RaidCatalogue(BaseCatalogue): SOURCE_PARSE = SOURCE_PARSE VALID_SOURCES = VALID_SOURCES - # ON DELETE CASCADE on revisions/encounters only fires with the - # per-connection FK pragma. - FOREIGN_KEYS = True - - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) - - def _create_schema(self, conn: sqlite3.Connection) -> None: - conn.execute(_SQL["schema_raid_zones"]) - conn.execute(_SQL["schema_raid_zone_revisions"]) - conn.execute(_SQL["schema_raid_encounters"]) - conn.execute(_SQL["schema_raid_encounter_revisions"]) - conn.execute(_SQL["schema_act_triggers"]) - conn.execute(_SQL["schema_act_spell_timers"]) - self._apply_migrations(conn, _ACT_ENRICHMENT_MIGRATIONS) - conn.executescript(_SQL["indexes_all"]) - - def _post_init(self, conn: sqlite3.Connection) -> None: - # One-time backfill (2026-08): the web ACT editor historically - # omitted modable / checked / the sound fields, so every save wrote - # the request-model defaults (0 / 0 / '') regardless of curator - # intent — synced EQ2Parser timers came out non-modable and silent. - # Flip the damage once; the meta guard means a curator's LATER - # deliberate zero/blank (now expressible in the editor) survives - # every re-init. - if get_meta(conn, "act_editor_parity_backfill"): - return - conn.execute("UPDATE act_spell_timers SET modable = 1") - conn.execute("UPDATE act_spell_timers SET checked = 1") - conn.execute( - "UPDATE act_spell_timers SET start_wav = 'tts', warning_wav = 'tts' " - "WHERE start_wav = '' AND warning_wav = ''" - ) - conn.commit() - set_meta(conn, "act_editor_parity_backfill", "1") + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) + + # ── Per-call read plumbing ─────────────────────────────────────────────── + + def _fetchall(self, sql: str, params: Sequence = ()) -> list[dict]: + """Run one read query on its own pooled checkout. Rows are dicts.""" + with self.init_db() as conn: + return conn.execute(sql, params).fetchall() + + def _fetchone(self, sql: str, params: Sequence = ()) -> dict | None: + """Single-row variant of :meth:`_fetchall`.""" + with self.init_db() as conn: + return conn.execute(sql, params).fetchone() # ── Write helpers (take an open conn — callers own the transaction) ────── @staticmethod def upsert_raid_zone( - conn: sqlite3.Connection, + conn: Any, *, zone_name: str, expansion_short: str, @@ -216,7 +180,7 @@ def upsert_raid_zone( existing = conn.execute(_SQL["select_zone_by_name"], (zone_name,)).fetchone() - if existing and source == SOURCE_SCRAPE and existing[1] == SOURCE_MANUAL: + if existing and source == SOURCE_SCRAPE and existing["source"] == SOURCE_MANUAL: # Re-scrape against a human-edited row: refresh the wiki-owned # metadata but leave the markdown blobs + source flag alone. The # revision history (encounters only) doesn't apply at the zone @@ -232,11 +196,11 @@ def upsert_raid_zone( lockout_min, lockout_max, now, - existing[0], + existing["id"], ), ) conn.commit() - return int(existing[0]) + return int(existing["id"]) # COALESCE on every nullable column so a caller that passes a column as # None means "don't touch", not "clobber to NULL". The historical default @@ -269,11 +233,11 @@ def upsert_raid_zone( ) conn.commit() row = conn.execute(_SQL["select_zone_id_by_name"], (zone_name,)).fetchone() - return int(row[0]) + return int(row["id"]) @staticmethod def upsert_raid_encounter( - conn: sqlite3.Connection, + conn: Any, *, raid_zone_id: int, mob_name: str, @@ -318,7 +282,7 @@ def upsert_raid_encounter( ), ) _row = cur.fetchone() - new_id = int(_row[0]) if _row else 0 + new_id = int(_row["id"]) if _row else 0 # First-ever revision row: before is NULL, after is the seeded content. if strategy_md is not None: conn.execute( @@ -328,14 +292,14 @@ def upsert_raid_encounter( conn.commit() return new_id - enc_id, prev_md = int(existing[0]), existing[1] + enc_id, prev_md = int(existing["id"]), existing["strategy_md"] # On re-scrape: only update fields that the scraper authoritatively # owns (wiki_url, position, last_synced_at). Don't clobber a # human-edited strategy_md with a fresh scrape — that's what # SOURCE_MANUAL exists to protect. if source == SOURCE_SCRAPE: - current_source = conn.execute(_SQL["select_encounter_source"], (enc_id,)).fetchone()[0] - if current_source == SOURCE_MANUAL: + src_row = conn.execute(_SQL["select_encounter_source"], (enc_id,)).fetchone() + if src_row is not None and src_row["source"] == SOURCE_MANUAL: # Refresh sync timestamp + url/position only, leave strategy alone. conn.execute( _SQL["update_encounter_url_position_synced"], @@ -374,11 +338,11 @@ def upsert_raid_encounter( conn.commit() return enc_id - # ── zones_db mirror helpers (no self-commit — callers commit) ──────────── + # ── zones mirror helpers (no self-commit — callers commit) ─────────────── @staticmethod def rename_raid_encounter_if_exists( - conn: sqlite3.Connection, + conn: Any, *, zone_name: str, old_mob_name: str, @@ -395,7 +359,7 @@ def rename_raid_encounter_if_exists( @staticmethod def update_raid_encounter_if_exists( - conn: sqlite3.Connection, + conn: Any, *, zone_name: str, mob_name: str, @@ -414,7 +378,7 @@ def update_raid_encounter_if_exists( return cur.rowcount > 0 @staticmethod - def delete_raid_encounter_by_zone_mob(conn: sqlite3.Connection, *, zone_name: str, mob_name: str) -> bool: + def delete_raid_encounter_by_zone_mob(conn: Any, *, zone_name: str, mob_name: str) -> bool: """Delete a raid_encounters row by its (zone_name, mob_name) lookup. CASCADEs to triggers, spell timers, strategy revisions via the FK. Returns True if a row was deleted.""" @@ -424,34 +388,32 @@ def delete_raid_encounter_by_zone_mob(conn: sqlite3.Connection, *, zone_name: st ) return cur.rowcount > 0 - # ── Read helpers (path-based) ──────────────────────────────────────────── + # ── Read helpers (own pooled checkout per call) ────────────────────────── def encounter_revisions(self, encounter_id: int) -> list[dict]: """Full revision history for an encounter, newest first.""" - return [dict(r) for r in self._fetchall(_SQL["list_encounter_revisions"], (encounter_id,))] + return self._fetchall(_SQL["list_encounter_revisions"], (encounter_id,)) def list_zone_revisions(self, zone_id: int) -> list[dict]: """All revision rows for a zone's overview, newest first. Each row: {id, edited_at, edited_by, before_md, after_md, edit_note}.""" - return [dict(r) for r in self._fetchall(_SQL["list_zone_revisions"], (zone_id,))] + return self._fetchall(_SQL["list_zone_revisions"], (zone_id,)) # ── ACT trigger helpers (formerly backend/eq2db/raids_act.py) ──────────── def list_act_triggers_for_encounter(self, encounter_id: int) -> list[dict]: """Every ACT trigger row for an encounter, ordered by position then id.""" - rows = self._fetchall( - f"SELECT {_ACT_TRIGGER_COLS} FROM act_triggers WHERE raid_encounter_id = ? ORDER BY position, id", + return self._fetchall( + f"SELECT {_ACT_TRIGGER_COLS} FROM act_triggers WHERE raid_encounter_id = %s ORDER BY position, id", (encounter_id,), ) - return [dict(r) for r in rows] def get_act_trigger(self, trigger_id: int) -> dict | None: - row = self._fetchone(f"SELECT {_ACT_TRIGGER_COLS} FROM act_triggers WHERE id = ?", (trigger_id,)) - return dict(row) if row else None + return self._fetchone(f"SELECT {_ACT_TRIGGER_COLS} FROM act_triggers WHERE id = %s", (trigger_id,)) @staticmethod def upsert_act_trigger( - conn: sqlite3.Connection, + conn: Any, *, trigger_id: int | None = None, raid_encounter_id: int, @@ -494,24 +456,24 @@ def upsert_act_trigger( category_restrict, category, timer, timer_name, tabbed, cooldown_seconds, last_edited_at, last_edited_by - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id """, params, ) _row = cur.fetchone() conn.commit() - return int(_row[0]) if _row else 0 + return int(_row["id"]) if _row else 0 conn.execute( """ UPDATE act_triggers SET - raid_encounter_id = ?, position = ?, label = ?, notes = ?, - active = ?, regex = ?, sound_data = ?, sound_type = ?, - category_restrict = ?, category = ?, - timer = ?, timer_name = ?, tabbed = ?, cooldown_seconds = ?, - last_edited_at = ?, last_edited_by = ? - WHERE id = ? + raid_encounter_id = %s, position = %s, label = %s, notes = %s, + active = %s, regex = %s, sound_data = %s, sound_type = %s, + category_restrict = %s, category = %s, + timer = %s, timer_name = %s, tabbed = %s, cooldown_seconds = %s, + last_edited_at = %s, last_edited_by = %s + WHERE id = %s """, params + (trigger_id,), ) @@ -519,9 +481,9 @@ def upsert_act_trigger( return trigger_id @staticmethod - def delete_act_trigger(conn: sqlite3.Connection, trigger_id: int) -> bool: + def delete_act_trigger(conn: Any, trigger_id: int) -> bool: """Delete a trigger by id. Returns True if a row was removed.""" - cur = conn.execute("DELETE FROM act_triggers WHERE id = ?", (trigger_id,)) + cur = conn.execute("DELETE FROM act_triggers WHERE id = %s", (trigger_id,)) conn.commit() return cur.rowcount > 0 @@ -529,19 +491,17 @@ def delete_act_trigger(conn: sqlite3.Connection, trigger_id: int) -> bool: def list_act_spell_timers_for_encounter(self, encounter_id: int) -> list[dict]: """Every spell-timer row for an encounter, alphabetical by name.""" - rows = self._fetchall( - f"SELECT {_ACT_SPELL_TIMER_COLS} FROM act_spell_timers WHERE raid_encounter_id = ? ORDER BY name", + return self._fetchall( + f"SELECT {_ACT_SPELL_TIMER_COLS} FROM act_spell_timers WHERE raid_encounter_id = %s ORDER BY name", (encounter_id,), ) - return [dict(r) for r in rows] def get_act_spell_timer(self, timer_id: int) -> dict | None: - row = self._fetchone(f"SELECT {_ACT_SPELL_TIMER_COLS} FROM act_spell_timers WHERE id = ?", (timer_id,)) - return dict(row) if row else None + return self._fetchone(f"SELECT {_ACT_SPELL_TIMER_COLS} FROM act_spell_timers WHERE id = %s", (timer_id,)) @staticmethod def upsert_act_spell_timer( - conn: sqlite3.Connection, + conn: Any, *, timer_id: int | None = None, raid_encounter_id: int, @@ -597,28 +557,29 @@ def upsert_act_spell_timer( category, restrict_category, damage_type, control_effect, last_edited_at, last_edited_by - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id """, params, ) _row = cur.fetchone() conn.commit() - return int(_row[0]) if _row else 0 + return int(_row["id"]) if _row else 0 conn.execute( """ UPDATE act_spell_timers SET - raid_encounter_id = ?, name = ?, name_lower = ?, - checked = ?, timer_duration_s = ?, - only_master_ticks = ?, restrict = ?, absolute_ = ?, - start_wav = ?, warning_wav = ?, warning_value = ?, - radial_display = ?, modable = ?, tooltip = ?, fill_color = ?, - panel1 = ?, panel2 = ?, remove_value = ?, - category = ?, restrict_category = ?, - damage_type = ?, control_effect = ?, - last_edited_at = ?, last_edited_by = ? - WHERE id = ? + raid_encounter_id = %s, name = %s, name_lower = %s, + checked = %s, timer_duration_s = %s, + only_master_ticks = %s, restrict = %s, absolute_ = %s, + start_wav = %s, warning_wav = %s, warning_value = %s, + radial_display = %s, modable = %s, tooltip = %s, fill_color = %s, + panel1 = %s, panel2 = %s, remove_value = %s, + category = %s, restrict_category = %s, + damage_type = %s, control_effect = %s, + last_edited_at = %s, last_edited_by = %s + WHERE id = %s """, params + (timer_id,), ) @@ -626,9 +587,9 @@ def upsert_act_spell_timer( return timer_id @staticmethod - def delete_act_spell_timer(conn: sqlite3.Connection, timer_id: int) -> bool: + def delete_act_spell_timer(conn: Any, timer_id: int) -> bool: """Delete a spell-timer by id. Returns True if a row was removed.""" - cur = conn.execute("DELETE FROM act_spell_timers WHERE id = ?", (timer_id,)) + cur = conn.execute("DELETE FROM act_spell_timers WHERE id = %s", (timer_id,)) conn.commit() return cur.rowcount > 0 diff --git a/backend/eq2db/raids.sql b/backend/eq2db/raids.sql index da9c5cb8..8c61c015 100644 --- a/backend/eq2db/raids.sql +++ b/backend/eq2db/raids.sql @@ -1,205 +1,27 @@ --- SQL for backend/eq2db/raids.py. Schema + DML both live here. The shared --- `_meta` table is created from backend/eq2db/_meta.sql via _meta.create_table(). - --- --------------------------------------------------------------------------- --- Schema --- --------------------------------------------------------------------------- - --- :name schema_raid_zones -CREATE TABLE IF NOT EXISTS raid_zones ( - -- Identity - id INTEGER PRIMARY KEY, - zone_name TEXT NOT NULL UNIQUE, -- matches zones.db zones.name - zone_name_lower TEXT NOT NULL, - - -- Denormalised from zones.db (intentional duplication so this DB - -- is queryable standalone; if the canonical changes, re-run the - -- scraper / sync job to refresh). - expansion_short TEXT NOT NULL, -- 'Vanilla' / 'DoF' / 'KoS' / 'EoF' / 'RoK' - wiki_url TEXT, - - -- Zone-level metadata extracted from the IZoneInformation template - -- on the wiki. All optional — missing fields just stay NULL. - access_md TEXT, -- how to get into the zone - background_md TEXT, -- lore / "Background" wiki section - overview_md TEXT, -- general zone-level tactics - level_range TEXT, -- e.g. '72-75' - zdiff TEXT, -- 'x4' / 'x2' / 'x3' - lockout_min TEXT, -- e.g. '2 days 20 hours' - lockout_max TEXT, -- e.g. '7 days' - - -- Audit trail - source TEXT NOT NULL, -- SOURCE_SCRAPE / SOURCE_MANUAL - last_synced_at INTEGER, -- unix ts of last wiki re-scrape - last_edited_at INTEGER, - last_edited_by TEXT -- discord_id or 'eq2i_scrape' -); - --- :name schema_raid_encounters -CREATE TABLE IF NOT EXISTS raid_encounters ( - id INTEGER PRIMARY KEY, - raid_zone_id INTEGER NOT NULL REFERENCES raid_zones(id) ON DELETE CASCADE, - mob_name TEXT NOT NULL, - mob_name_lower TEXT NOT NULL, - position INTEGER NOT NULL DEFAULT 0, -- order within the zone - - -- Free-form markdown strategy. Single blob deliberately — PoC - -- simplicity. If a structured pattern emerges (cures, dispels, - -- phases) we can split later without breaking callers. - strategy_md TEXT, - - wiki_url TEXT, - source TEXT NOT NULL, - last_synced_at INTEGER, - last_edited_at INTEGER, - last_edited_by TEXT, - - UNIQUE (raid_zone_id, mob_name_lower) -); - --- :name schema_raid_encounter_revisions -CREATE TABLE IF NOT EXISTS raid_encounter_revisions ( - id INTEGER PRIMARY KEY, - encounter_id INTEGER NOT NULL REFERENCES raid_encounters(id) ON DELETE CASCADE, - edited_at INTEGER NOT NULL, - edited_by TEXT NOT NULL, -- discord_id or scrape token - before_md TEXT, -- previous strategy_md (NULL on create) - after_md TEXT NOT NULL, -- new strategy_md - edit_note TEXT -- optional commit-message style note -); - --- :name schema_raid_zone_revisions -CREATE TABLE IF NOT EXISTS raid_zone_revisions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raid_zone_id INTEGER NOT NULL, - edited_at INTEGER NOT NULL, - edited_by TEXT NOT NULL, -- discord_id or scrape token - before_md TEXT, -- NULL on the very first row (seed) - after_md TEXT NOT NULL, - edit_note TEXT, -- optional commit-style note - FOREIGN KEY (raid_zone_id) REFERENCES raid_zones (id) ON DELETE CASCADE -); - --- ACT Triggers — regex-driven matchers a player imports into Advanced Combat --- Tracker to react to in-game log lines (boss callouts, debuffs, mechanic --- triggers). One row maps 1:1 to a element in ACT's --- spell_timers.xml export format (column names mirror XML attributes via --- snake_case). --- --- A trigger with timer=1 references an entry in act_spell_timers by --- timer_name; on XML export both rows are emitted so the dropped file --- round-trips in ACT without manual fix-up. --- :name schema_act_triggers -CREATE TABLE IF NOT EXISTS act_triggers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raid_encounter_id INTEGER NOT NULL REFERENCES raid_encounters(id) ON DELETE CASCADE, - - -- Display / curation (web-only — no XML counterpart) - position INTEGER NOT NULL DEFAULT 0, -- ordering within encounter - label TEXT, -- human-readable summary line; falls back to sound_data/regex preview - notes TEXT, -- contributor explanation, never exported - - -- ACT attributes (9 fields) - active INTEGER NOT NULL DEFAULT 1, - regex TEXT NOT NULL, - sound_data TEXT NOT NULL DEFAULT '', - sound_type INTEGER NOT NULL DEFAULT 3, -- 3 = TTS, 0 = silent / file - category_restrict INTEGER NOT NULL DEFAULT 0, - category TEXT, -- defaults to mob_name at write time - timer INTEGER NOT NULL DEFAULT 0, - timer_name TEXT, -- loose name-FK into act_spell_timers (same encounter) - tabbed INTEGER NOT NULL DEFAULT 0, - - -- EQ2Parser enrichment (served by /api/act/pack — NEVER exported to - -- ACT XML; plain ACT stays byte-compatible) - cooldown_seconds REAL NOT NULL DEFAULT 1.0, -- per-trigger audio cooldown (ACT hardcodes 1s) - - -- Audit - last_edited_at INTEGER, - last_edited_by TEXT, - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) -); - --- ACT Spell Timers — named timer definitions referenced by act_triggers --- via timer_name. One row maps 1:1 to a element in ACT's --- spell_timers.xml. Multiple triggers MAY reference the same timer name --- within an encounter (DRY); export deduplicates by name. --- :name schema_act_spell_timers -CREATE TABLE IF NOT EXISTS act_spell_timers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raid_encounter_id INTEGER NOT NULL REFERENCES raid_encounters(id) ON DELETE CASCADE, - - -- Identity (Name is what triggers reference via TimerName) - name TEXT NOT NULL, - name_lower TEXT NOT NULL, - - -- ACT attributes (17 fields) - checked INTEGER NOT NULL DEFAULT 0, - timer_duration_s INTEGER NOT NULL, -- "Timer" attribute in XML - only_master_ticks INTEGER NOT NULL DEFAULT 0, - restrict INTEGER NOT NULL DEFAULT 0, - absolute_ INTEGER NOT NULL DEFAULT 0, -- "Absolute" — column name disambiguated from SQL keyword - start_wav TEXT NOT NULL DEFAULT '', - warning_wav TEXT NOT NULL DEFAULT '', - warning_value INTEGER NOT NULL DEFAULT 10, - radial_display INTEGER NOT NULL DEFAULT 0, - modable INTEGER NOT NULL DEFAULT 0, - tooltip TEXT NOT NULL DEFAULT '', - fill_color INTEGER NOT NULL DEFAULT -16776961, -- ACT default blue (.NET ARGB packed int) - panel1 INTEGER NOT NULL DEFAULT 1, - panel2 INTEGER NOT NULL DEFAULT 0, - remove_value INTEGER NOT NULL DEFAULT -15, - category TEXT, -- defaults to mob_name at write time - restrict_category INTEGER NOT NULL DEFAULT 0, - - -- EQ2Parser enrichment (served by /api/act/pack — NEVER exported to - -- ACT XML; plain ACT stays byte-compatible). Closed vocabularies — - -- damage_type is comma-joined dominant-first log schools ("poison, - -- disease"); control_effect is tooltip vocabulary ("stifle", "stun"). - damage_type TEXT NOT NULL DEFAULT '', - control_effect TEXT NOT NULL DEFAULT '', - - -- Audit - last_edited_at INTEGER, - last_edited_by TEXT, - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - - UNIQUE (raid_encounter_id, name_lower) -); - --- Multi-statement block — init_db runs via conn.executescript(). --- :name indexes_all -CREATE INDEX IF NOT EXISTS idx_raid_zones_name_lower ON raid_zones (zone_name_lower); -CREATE INDEX IF NOT EXISTS idx_raid_zones_expansion ON raid_zones (expansion_short); -CREATE INDEX IF NOT EXISTS idx_raid_enc_zone ON raid_encounters (raid_zone_id, position); -CREATE INDEX IF NOT EXISTS idx_raid_enc_mob_lower ON raid_encounters (mob_name_lower); -CREATE INDEX IF NOT EXISTS idx_raid_rev_encounter ON raid_encounter_revisions (encounter_id, edited_at); -CREATE INDEX IF NOT EXISTS idx_raid_zone_rev_zone ON raid_zone_revisions (raid_zone_id, edited_at); -CREATE INDEX IF NOT EXISTS idx_act_triggers_enc ON act_triggers (raid_encounter_id, position); -CREATE INDEX IF NOT EXISTS idx_act_triggers_timer ON act_triggers (raid_encounter_id, timer_name); -CREATE INDEX IF NOT EXISTS idx_act_spell_timers_enc ON act_spell_timers (raid_encounter_id); +-- SQL for backend/eq2db/raids.py (psycopg, raids schema). DML only — +-- schema DDL lives in db/migrations/0005_raids.sql. -- --------------------------------------------------------------------------- -- raid_zones -- --------------------------------------------------------------------------- -- :name select_zone_by_name -SELECT id, source FROM raid_zones WHERE zone_name = ?; +SELECT id, source FROM raid_zones WHERE zone_name = %s; -- :name select_zone_id_by_name -SELECT id FROM raid_zones WHERE zone_name = ?; +SELECT id FROM raid_zones WHERE zone_name = %s; -- Refresh wiki-owned columns on an existing human-edited row. -- :name update_zone_wiki_fields UPDATE raid_zones SET - expansion_short = ?, - wiki_url = ?, - level_range = ?, - zdiff = ?, - lockout_min = ?, - lockout_max = ?, - last_synced_at = ? -WHERE id = ?; + expansion_short = %s, + wiki_url = %s, + level_range = %s, + zdiff = %s, + lockout_min = %s, + lockout_max = %s, + last_synced_at = %s +WHERE id = %s; -- COALESCE-on-conflict so a None caller param means "don't touch", not -- "clobber to NULL". See the long comment in raids.py:upsert_raid_zone for @@ -211,7 +33,7 @@ INSERT INTO raid_zones ( access_md, background_md, overview_md, level_range, zdiff, lockout_min, lockout_max, source, last_synced_at -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT(zone_name) DO UPDATE SET expansion_short = COALESCE(excluded.expansion_short, raid_zones.expansion_short), wiki_url = COALESCE(excluded.wiki_url, raid_zones.wiki_url), @@ -230,60 +52,62 @@ ON CONFLICT(zone_name) DO UPDATE SET -- --------------------------------------------------------------------------- -- :name select_encounter_by_zone_mob -SELECT id, strategy_md FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?; +SELECT id, strategy_md FROM raid_encounters WHERE raid_zone_id = %s AND mob_name_lower = %s; -- :name select_encounter_source -SELECT source FROM raid_encounters WHERE id = ?; +SELECT source FROM raid_encounters WHERE id = %s; -- :name insert_encounter INSERT INTO raid_encounters ( raid_zone_id, mob_name, mob_name_lower, position, strategy_md, wiki_url, source, last_synced_at, last_edited_at, last_edited_by -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id; -- Refresh sync timestamp + URL + position on a manually-edited row, leaving -- strategy_md alone — re-scrape doesn't overwrite curator edits. -- :name update_encounter_url_position_synced -UPDATE raid_encounters SET wiki_url = ?, position = ?, last_synced_at = ? WHERE id = ?; +UPDATE raid_encounters SET wiki_url = %s, position = %s, last_synced_at = %s WHERE id = %s; -- Big conditional update: only the SCRAPE actor refreshes last_synced_at; -- only non-SCRAPE actors stamp last_edited_at + last_edited_by. The CASE --- WHEN guards keep the wrong column from being clobbered. +-- WHEN guards keep the wrong column from being clobbered. The ::text casts +-- let Postgres type the parameter-to-parameter comparisons (both sides are +-- source tokens). -- :name update_encounter UPDATE raid_encounters SET - mob_name = ?, - position = ?, - strategy_md = COALESCE(?, strategy_md), - wiki_url = ?, - source = ?, - last_synced_at = CASE WHEN ?=? THEN ? ELSE last_synced_at END, - last_edited_at = CASE WHEN ?<>? THEN ? ELSE last_edited_at END, - last_edited_by = CASE WHEN ?<>? THEN ? ELSE last_edited_by END -WHERE id = ?; + mob_name = %s, + position = %s, + strategy_md = COALESCE(%s, strategy_md), + wiki_url = %s, + source = %s, + last_synced_at = CASE WHEN %s::text = %s::text THEN %s ELSE last_synced_at END, + last_edited_at = CASE WHEN %s::text <> %s::text THEN %s ELSE last_edited_at END, + last_edited_by = CASE WHEN %s::text <> %s::text THEN %s ELSE last_edited_by END +WHERE id = %s; -- :name rename_encounter_by_zone_mob UPDATE raid_encounters - SET mob_name = ?, - mob_name_lower = ?, - last_edited_at = strftime('%s','now') + SET mob_name = %s, + mob_name_lower = %s, + last_edited_at = floor(extract(epoch from now()))::bigint WHERE id IN ( SELECT re.id FROM raid_encounters re JOIN raid_zones rz ON rz.id = re.raid_zone_id - WHERE rz.zone_name_lower = ? - AND re.mob_name_lower = ? + WHERE rz.zone_name_lower = %s + AND re.mob_name_lower = %s ); -- :name update_encounter_position_by_zone_mob UPDATE raid_encounters - SET position = ?, - last_edited_at = strftime('%s','now') + SET position = %s, + last_edited_at = floor(extract(epoch from now()))::bigint WHERE id IN ( SELECT re.id FROM raid_encounters re JOIN raid_zones rz ON rz.id = re.raid_zone_id - WHERE rz.zone_name_lower = ? - AND re.mob_name_lower = ? + WHERE rz.zone_name_lower = %s + AND re.mob_name_lower = %s ); -- :name delete_encounter_by_zone_mob @@ -291,20 +115,20 @@ DELETE FROM raid_encounters WHERE id IN ( SELECT re.id FROM raid_encounters re JOIN raid_zones rz ON rz.id = re.raid_zone_id - WHERE rz.zone_name_lower = ? - AND re.mob_name_lower = ? + WHERE rz.zone_name_lower = %s + AND re.mob_name_lower = %s ); -- --------------------------------------------------------------------------- -- raid_encounter_revisions -- --------------------------------------------------------------------------- --- before_md may be NULL (first-ever revision) — caller passes None and SQLite --- handles the binding. One INSERT covers both seeding and updates. +-- before_md may be NULL (first-ever revision) — caller passes None and the +-- driver handles the binding. One INSERT covers both seeding and updates. -- :name insert_encounter_revision INSERT INTO raid_encounter_revisions (encounter_id, edited_at, edited_by, before_md, after_md, edit_note) -VALUES (?, ?, ?, ?, ?, ?); +VALUES (%s, %s, %s, %s, %s, %s); -- --------------------------------------------------------------------------- -- Read helpers @@ -313,9 +137,9 @@ VALUES (?, ?, ?, ?, ?, ?); -- :name list_encounter_revisions SELECT id, encounter_id, edited_at, edited_by, before_md, after_md, edit_note FROM raid_encounter_revisions -WHERE encounter_id = ? ORDER BY edited_at DESC, id DESC; +WHERE encounter_id = %s ORDER BY edited_at DESC, id DESC; -- :name list_zone_revisions SELECT id, edited_at, edited_by, before_md, after_md, edit_note -FROM raid_zone_revisions WHERE raid_zone_id = ? +FROM raid_zone_revisions WHERE raid_zone_id = %s ORDER BY edited_at DESC, id DESC; diff --git a/backend/eq2db/zones.py b/backend/eq2db/zones.py index 478bc0f2..a0857986 100644 --- a/backend/eq2db/zones.py +++ b/backend/eq2db/zones.py @@ -1,12 +1,13 @@ """ -Local SQLite catalogue of EverQuest 2 zones. +Postgres catalogue of EverQuest 2 zones (the ``zones`` schema). Sourced from ``scripts/dev/eq2_zones.cleaned.json`` (produced by ``scripts/dev/clean_eq2_zones.py`` from a noisy EQ2 wiki dump). Run -``scripts/build_zones_db.py`` to (re)build the DB after the cleaned JSON -changes — idempotent. +``scripts/build_zones_db.py`` to (re)load the zone metadata after the +cleaned JSON changes — idempotent, and it never touches the curator-managed +``zone_encounters`` / ``zone_encounter_mobs`` / ``featured_*`` tables. -Schema (five tables): +Schema (db/migrations/0004_zones.sql — migrations own the DDL): * **zones** — one row per canonical zone with classification. @@ -33,18 +34,16 @@ import ONE name, the shared ``catalogue`` instance. The frozen dataclass models (Zone, ZoneEncounter, ZoneEncounterMob, FeaturedRaid*) stay as the typed active-record layer underneath; catalogue methods delegate to them -with the instance's ``path`` and return the legacy dict shapes the routes +with the instance's ``schema`` and return the legacy dict shapes the routes consume. """ from __future__ import annotations -import sqlite3 from dataclasses import dataclass -from pathlib import Path +from typing import Any -from backend.db_catalogue import BaseCatalogue -from backend.db_helpers import resolve_db_path +from backend.db_catalogue import PgCatalogue, PgConnProxy from backend.sql_loader import load_sql _SQL = load_sql(__file__) @@ -54,16 +53,37 @@ # --------------------------------------------------------------------------- -DB_PATH: Path = resolve_db_path("DB_ZONES_PATH", "zones", "zones.db") +#: Postgres schema the zones family lives in (db/migrations/0004_zones.sql). +SCHEMA = "zones" -# Schema (CREATE TABLE / INDEX) lives in zones.sql; init_db runs each block. # Column-list fragment for find_* queries is loaded from zones.sql at module import. _SELECT_COLS = _SQL["select_zone_cols"] -# `_meta` get/set is shared across every eq2db module — see backend/eq2db/_meta.py. -from backend.eq2db._meta import get_meta, set_meta # noqa: E402,F401 +def _connect(schema: str) -> PgConnProxy: + """One pooled, schema-scoped connection — the model classmethods' analog + of the old ``sqlite3.connect(path)``. ``with _connect(schema) as conn:`` + commits on clean exit, rolls back on exception, then returns the + connection to the pool.""" + return PgConnProxy(schema) + + +# zones-schema `_meta` helpers. The shared backend/eq2db/_meta.py helper +# keeps its `?` dialect for the SQLite catalogues (items/spells/recipes); +# zones routes its provenance reads/writes through these %s-dialect blocks. +def get_meta(conn: Any, key: str, default: str | None = None) -> str | None: + """Return the value for ``key`` or ``default`` if missing.""" + row = conn.execute(_SQL["meta_select"], (key,)).fetchone() + return row["value"] if row else default + + +def set_meta(conn: Any, key: str, value: str) -> None: + """Upsert ``(key, value)``. Commits immediately — meta writes are + one-shot and don't compose into larger transactions.""" + conn.execute(_SQL["meta_upsert"], (key, value)) + conn.commit() + # --------------------------------------------------------------------------- # Zone active-record model @@ -71,8 +91,7 @@ # The Zone dataclass owns every column from the `zones` table plus the # eagerly-loaded child collections (types, aliases, bosses). Classmethods # replace the old free-function lookups; instance methods cover the -# `zone_types` side-table mutations. Free-function shims below preserve -# the historical dict-returning contract so routes + tests work unchanged. +# `zone_types` side-table mutations. @dataclass(frozen=True) @@ -113,10 +132,9 @@ class Zone: bosses: list[ZoneEncounter] @classmethod - def _from_row(cls, conn: sqlite3.Connection, row: sqlite3.Row) -> Zone: - """Build a Zone from a `zones`-table row plus child fetches. - Caller must have set ``conn.row_factory = sqlite3.Row`` and the - row must carry every column in `_SELECT_COLS` (the canonical + def _from_row(cls, conn: Any, row: Any) -> Zone: + """Build a Zone from a `zones`-table row (dict) plus child fetches. + The row must carry every column in `_SELECT_COLS` (the canonical column list at the top of zones.sql).""" return cls( id=row["id"], @@ -139,8 +157,8 @@ def _from_row(cls, conn: sqlite3.Connection, row: sqlite3.Row) -> Zone: is_deprecated=bool(row["is_deprecated"]), event_name=row["event_name"], wiki_url=row["wiki_url"], - types=[r[0] for r in conn.execute(_SQL["list_types_for_zone"], (row["id"],))], - aliases=[r[0] for r in conn.execute(_SQL["list_aliases_for_zone"], (row["id"],))], + types=[r["type"] for r in conn.execute(_SQL["list_types_for_zone"], (row["id"],))], + aliases=[r["alias"] for r in conn.execute(_SQL["list_aliases_for_zone"], (row["id"],))], bosses=ZoneEncounter._list_for_zone_id(conn, row["id"]), ) @@ -180,7 +198,7 @@ def to_dict(self) -> dict: # ----------------------------------------------------------------- @classmethod - def find_by_name(cls, name: str, *, path: Path) -> Zone | None: + def find_by_name(cls, name: str, *, schema: str) -> Zone | None: """Resolve a zone by name, falling back to the alias table. Lookup order: @@ -189,10 +207,9 @@ def find_by_name(cls, name: str, *, path: Path) -> Zone | None: ACT log lookups should use this — the alias table covers the "with-The vs without-The" wiki dup pairs.""" - if not path.exists() or not name: + if not name: return None - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: row = conn.execute( _SQL["find_zone_by_name_lower"].format(cols=_SELECT_COLS), (name.lower(),), @@ -203,7 +220,7 @@ def find_by_name(cls, name: str, *, path: Path) -> Zone | None: return None row = conn.execute( _SQL["find_zone_by_id"].format(cols=_SELECT_COLS), - (alias_row[0],), + (alias_row["zone_id"],), ).fetchone() if row is None: return None # orphaned alias — shouldn't happen with FKs @@ -215,14 +232,11 @@ def list_by_expansion( expansion_short: str, *, type_filter: str | None = None, - path: Path, + schema: str, ) -> list[Zone]: """All zones in an expansion, ordered by name. Optionally filter to a single type token (e.g. 'raid_x4', 'group', 'tradeskill').""" - if not path.exists(): - return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: if type_filter: rows = conn.execute( _SQL["list_zones_by_expansion_typed"].format(cols=_SELECT_COLS), @@ -236,12 +250,9 @@ def list_by_expansion( return [cls._from_row(conn, r) for r in rows] @classmethod - def list_by_event(cls, event_name: str, *, path: Path) -> list[Zone]: + def list_by_event(cls, event_name: str, *, schema: str) -> list[Zone]: """All zones for a recurring in-game event (Tinkerfest, Frostfell).""" - if not path.exists(): - return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: rows = conn.execute( _SQL["list_zones_by_event"].format(cols=_SELECT_COLS), (event_name,), @@ -249,12 +260,9 @@ def list_by_event(cls, event_name: str, *, path: Path) -> list[Zone]: return [cls._from_row(conn, r) for r in rows] @classmethod - def list_by_type(cls, type_token: str, *, path: Path) -> list[Zone]: + def list_by_type(cls, type_token: str, *, schema: str) -> list[Zone]: """All zones tagged with a given type token across all expansions.""" - if not path.exists(): - return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: rows = conn.execute( _SQL["list_zones_by_type"].format(cols=_SELECT_COLS), (type_token,), @@ -262,7 +270,7 @@ def list_by_type(cls, type_token: str, *, path: Path) -> list[Zone]: return [cls._from_row(conn, r) for r in rows] @classmethod - def find_by_boss(cls, mob_name: str, *, path: Path) -> list[Zone]: + def find_by_boss(cls, mob_name: str, *, schema: str) -> list[Zone]: """Reverse lookup: which zone(s) host a given raid boss? Joins through zone_encounter_mobs so individual mob names inside @@ -271,10 +279,9 @@ def find_by_boss(cls, mob_name: str, *, path: Path) -> list[Zone]: Returns a list because the same mob name can appear in multiple zones (Fabled variants, multi-instance bosses).""" - if not path.exists() or not mob_name: + if not mob_name: return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: rows = conn.execute( _SQL["list_zones_by_boss"].format(cols=_SELECT_COLS), (mob_name.lower(),), @@ -285,15 +292,13 @@ def find_by_boss(cls, mob_name: str, *, path: Path) -> list[Zone]: # zone_types mutations # ----------------------------------------------------------------- - def add_type(self, type_token: str, *, path: Path) -> Zone: + def add_type(self, type_token: str, *, schema: str) -> Zone: """Add a type tag (e.g. 'dungeon') to this zone. Idempotent — - adding the same tag twice is a no-op (INSERT OR IGNORE against - the PK). Returns a freshly-loaded Zone reflecting the new tag - list (the frozen dataclass is immutable, so the in-memory ``self`` - is left untouched).""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") + adding the same tag twice is a no-op (ON CONFLICT DO NOTHING + against the PK). Returns a freshly-loaded Zone reflecting the new + tag list (the frozen dataclass is immutable, so the in-memory + ``self`` is left untouched).""" + with _connect(schema) as conn: conn.execute(_SQL["insert_zone_type_or_ignore"], (self.id, type_token)) conn.commit() row = conn.execute( @@ -304,13 +309,11 @@ def add_type(self, type_token: str, *, path: Path) -> Zone: # we just held it in memory; nothing deletes it concurrently. return Zone._from_row(conn, row) - def remove_type(self, type_token: str, *, path: Path) -> Zone: + def remove_type(self, type_token: str, *, schema: str) -> Zone: """Remove a type tag from this zone. Idempotent — a no-op when the tag isn't present. Returns a freshly-loaded Zone reflecting the updated tag list.""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") + with _connect(schema) as conn: conn.execute(_SQL["delete_zone_type"], (self.id, type_token)) conn.commit() row = conn.execute( @@ -325,7 +328,7 @@ def remove_type(self, type_token: str, *, path: Path) -> Zone: # --------------------------------------------------------------------------- -def _hydrate_zone(conn: sqlite3.Connection, row: sqlite3.Row) -> dict: +def _hydrate_zone(conn: Any, row: Any) -> dict: """Build the legacy hydrated-zone dict from a `zones`-table row + side queries for types/aliases/bosses. Now a thin wrapper around ``Zone._from_row(conn, row).to_dict()`` — same SQL, same shape, no @@ -340,20 +343,20 @@ def _hydrate_zone(conn: sqlite3.Connection, row: sqlite3.Row) -> dict: # --------------------------------------------------------------------------- -def _zone_name_and_expansion(zone_id: int, path: Path) -> tuple[str | None, str | None]: +def _zone_name_and_expansion(zone_id: int, schema: str) -> tuple[str | None, str | None]: """Canonical zone name + expansion for the raids_db mirror.""" - with sqlite3.connect(path) as conn: + with _connect(schema) as conn: r = conn.execute(_SQL["select_zone_name_and_expansion"], (zone_id,)).fetchone() - return (r[0], r[1]) if r else (None, None) + return (r["name"], r["expansion_short"]) if r else (None, None) -def _mirror_primary_rename_in_raids_db(zone_id: int, old_name: str, new_name: str, path: Path) -> None: +def _mirror_primary_rename_in_raids_db(zone_id: int, old_name: str, new_name: str, schema: str) -> None: """Rename a raids_db.raid_encounters row keyed by (zone_name, old_name) → new_name, if it exists. Looks up zone_name from the parent encounter. Used by both ZoneEncounter.update() and ZoneEncounterMob.rename().""" if old_name == new_name: return - zone_name, _exp = _zone_name_and_expansion(zone_id, path) + zone_name, _exp = _zone_name_and_expansion(zone_id, schema) if zone_name is None: return from backend.eq2db.raids import catalogue as _raids_db @@ -387,8 +390,8 @@ class ZoneEncounterMob: position: int @classmethod - def _from_row(cls, row: sqlite3.Row, *, encounter_id: int | None = None) -> ZoneEncounterMob: - """Build from a sqlite3.Row. ``encounter_id`` is taken from the row + def _from_row(cls, row: Any, *, encounter_id: int | None = None) -> ZoneEncounterMob: + """Build from a dict row. ``encounter_id`` is taken from the row when present, otherwise from the caller (e.g. when the SELECT doesn't bring it back because the caller already knows it).""" keys = row.keys() @@ -406,7 +409,7 @@ def add_to_encounter( *, mob_name: str, make_primary: bool = False, - path: Path, + schema: str, ) -> ZoneEncounterMob: """Add a mob to an encounter. By default appends as a sibling at the next available position. With ``make_primary=True``, shifts every @@ -414,42 +417,40 @@ def add_to_encounter( updates the parent encounter_name to the new primary (mirrored to raids_db).""" old_primary_name: str | None = None - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") - with conn: - if make_primary: - # Capture the current primary's name BEFORE the shift, - # so we know what to rename in raids_db. - primary = conn.execute(_SQL["select_primary_mob_name"], (encounter_id,)).fetchone() - if primary is not None: - old_primary_name = primary["mob_name"] - # Two-phase shift of existing mobs down by 1. - conn.execute(_SQL["shift_mobs_negative"], (encounter_id,)) - conn.execute(_SQL["shift_mobs_back_positive"], (encounter_id,)) - cur = conn.execute( - _SQL["insert_encounter_mob_primary"], - (encounter_id, mob_name, mob_name.lower()), - ) - new_id = (cur.fetchone() or (None,))[0] - conn.execute(_SQL["update_encounter_name"], (mob_name, encounter_id)) - else: - next_pos = conn.execute(_SQL["max_mob_position_for_encounter"], (encounter_id,)).fetchone()[0] - cur = conn.execute( - _SQL["insert_encounter_mob"], - (encounter_id, mob_name, mob_name.lower(), next_pos), - ) - new_id = (cur.fetchone() or (None,))[0] + with _connect(schema) as conn: + if make_primary: + # Capture the current primary's name BEFORE the shift, + # so we know what to rename in raids_db. + primary = conn.execute(_SQL["select_primary_mob_name"], (encounter_id,)).fetchone() + if primary is not None: + old_primary_name = primary["mob_name"] + # Two-phase shift of existing mobs down by 1. + conn.execute(_SQL["shift_mobs_negative"], (encounter_id,)) + conn.execute(_SQL["shift_mobs_back_positive"], (encounter_id,)) + cur = conn.execute( + _SQL["insert_encounter_mob_primary"], + (encounter_id, mob_name, mob_name.lower()), + ) + new_id = PgCatalogue.fetchval(cur) + conn.execute(_SQL["update_encounter_name"], (mob_name, encounter_id)) + else: + next_pos = PgCatalogue.fetchval(conn.execute(_SQL["max_mob_position_for_encounter"], (encounter_id,))) + cur = conn.execute( + _SQL["insert_encounter_mob"], + (encounter_id, mob_name, mob_name.lower(), next_pos), + ) + new_id = PgCatalogue.fetchval(cur) row = conn.execute(_SQL["select_mob_by_id"], (new_id,)).fetchone() if make_primary and old_primary_name is not None: - _mirror_primary_rename_in_raids_db(_encounter_zone_id(encounter_id, path), old_primary_name, mob_name, path) + _mirror_primary_rename_in_raids_db( + _encounter_zone_id(encounter_id, schema), old_primary_name, mob_name, schema + ) return cls._from_row(row, encounter_id=encounter_id) @classmethod - def find_by_id(cls, mob_id: int, *, path: Path) -> ZoneEncounterMob | None: + def find_by_id(cls, mob_id: int, *, schema: str) -> ZoneEncounterMob | None: """Single-mob fetch by id. Returns None if not found.""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: row = conn.execute(_SQL["select_mob_for_update"], (mob_id,)).fetchone() return ( cls(id=mob_id, encounter_id=row["encounter_id"], mob_name=row["mob_name"], position=row["position"]) @@ -458,79 +459,69 @@ def find_by_id(cls, mob_id: int, *, path: Path) -> ZoneEncounterMob | None: ) @classmethod - def list_for_encounter(cls, encounter_id: int, *, path: Path) -> list[ZoneEncounterMob]: + def list_for_encounter(cls, encounter_id: int, *, schema: str) -> list[ZoneEncounterMob]: """All mobs for an encounter, ordered by position.""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: return [ cls._from_row(r, encounter_id=encounter_id) for r in conn.execute(_SQL["list_mobs_for_encounter_asc"], (encounter_id,)) ] - def rename(self, new_mob_name: str, *, path: Path) -> ZoneEncounterMob: + def rename(self, new_mob_name: str, *, schema: str) -> ZoneEncounterMob: """Rename. If this mob is at position 0 (the primary), also updates the parent encounter_name so the two stay in sync, and mirrors the rename onto raids_db.raid_encounters (if a row exists there). Returns the renamed instance.""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") - with conn: - conn.execute(_SQL["update_mob_name"], (new_mob_name, new_mob_name.lower(), self.id)) - if self.position == 0: - conn.execute(_SQL["update_encounter_name"], (new_mob_name, self.encounter_id)) + with _connect(schema) as conn: + conn.execute(_SQL["update_mob_name"], (new_mob_name, new_mob_name.lower(), self.id)) + if self.position == 0: + conn.execute(_SQL["update_encounter_name"], (new_mob_name, self.encounter_id)) if self.position == 0: _mirror_primary_rename_in_raids_db( - _encounter_zone_id(self.encounter_id, path), self.mob_name, new_mob_name, path + _encounter_zone_id(self.encounter_id, schema), self.mob_name, new_mob_name, schema ) return ZoneEncounterMob( id=self.id, encounter_id=self.encounter_id, mob_name=new_mob_name, position=self.position ) - def promote_to_primary(self, *, path: Path) -> ZoneEncounterMob: + def promote_to_primary(self, *, schema: str) -> ZoneEncounterMob: """Swap this mob (a sibling) with the current primary (position 0). No-op if already primary. Updates the parent encounter_name and mirrors the rename onto raids_db. Returns the promoted instance.""" if self.position == 0: return self - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") + with _connect(schema) as conn: primary = conn.execute(_SQL["select_primary_mob_id_and_name"], (self.encounter_id,)).fetchone() if primary is None: # Shouldn't happen if invariants hold, but defensively: just # move this mob to position 0 with no swap. - with conn: - conn.execute(_SQL["update_mob_position_to_zero"], (self.id,)) - conn.execute(_SQL["update_encounter_name"], (self.mob_name, self.encounter_id)) - return ZoneEncounterMob(id=self.id, encounter_id=self.encounter_id, mob_name=self.mob_name, position=0) - old_primary_name = primary["mob_name"] - with conn: - # Park the old primary at -1 (sentinel), promote the sibling - # to 0, then move the old primary into the sibling's old slot. - conn.execute(_SQL["update_mob_position_to_neg_one"], (primary["id"],)) conn.execute(_SQL["update_mob_position_to_zero"], (self.id,)) - conn.execute(_SQL["update_mob_position"], (self.position, primary["id"])) conn.execute(_SQL["update_encounter_name"], (self.mob_name, self.encounter_id)) + return ZoneEncounterMob(id=self.id, encounter_id=self.encounter_id, mob_name=self.mob_name, position=0) + old_primary_name = primary["mob_name"] + # Park the old primary at -1 (sentinel), promote the sibling + # to 0, then move the old primary into the sibling's old slot. + conn.execute(_SQL["update_mob_position_to_neg_one"], (primary["id"],)) + conn.execute(_SQL["update_mob_position_to_zero"], (self.id,)) + conn.execute(_SQL["update_mob_position"], (self.position, primary["id"])) + conn.execute(_SQL["update_encounter_name"], (self.mob_name, self.encounter_id)) _mirror_primary_rename_in_raids_db( - _encounter_zone_id(self.encounter_id, path), old_primary_name, self.mob_name, path + _encounter_zone_id(self.encounter_id, schema), old_primary_name, self.mob_name, schema ) return ZoneEncounterMob(id=self.id, encounter_id=self.encounter_id, mob_name=self.mob_name, position=0) - def delete(self, *, path: Path) -> bool: + def delete(self, *, schema: str) -> bool: """Delete this mob. Refuses with ValueError when it's the only mob in the encounter (an encounter needs ≥ 1 mob) or when it's the primary while siblings exist (caller must promote a sibling first). Returns False if the row is no longer present, True on successful delete.""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") + with _connect(schema) as conn: row = conn.execute(_SQL["select_mob_encounter_position"], (self.id,)).fetchone() if row is None: return False - total = conn.execute(_SQL["count_mobs_for_encounter"], (row["encounter_id"],)).fetchone()[0] - if total <= 1: + total = PgCatalogue.fetchval(conn.execute(_SQL["count_mobs_for_encounter"], (row["encounter_id"],))) + if total is None or total <= 1: raise ValueError("cannot delete the last mob of an encounter") if row["position"] == 0: raise ValueError( @@ -541,12 +532,12 @@ def delete(self, *, path: Path) -> bool: return True -def _encounter_zone_id(encounter_id: int, path: Path) -> int: +def _encounter_zone_id(encounter_id: int, schema: str) -> int: """Cheap zone_id lookup off an encounter — used by the raids-db mirror paths to resolve zone_name without re-reading the full encounter row.""" - with sqlite3.connect(path) as conn: + with _connect(schema) as conn: row = conn.execute(_SQL["select_encounter_zone_id"], (encounter_id,)).fetchone() - return int(row[0]) if row else 0 + return int(row["zone_id"]) if row else 0 @dataclass(frozen=True) @@ -564,7 +555,7 @@ class ZoneEncounter: mobs: list[ZoneEncounterMob] @classmethod - def _from_row(cls, conn: sqlite3.Connection, row: sqlite3.Row) -> ZoneEncounter: + def _from_row(cls, conn: Any, row: Any) -> ZoneEncounter: """Build from an encounter row + a fresh mob fetch using the open connection. Mobs come back with their real ids so downstream code can mutate them directly via ``ZoneEncounterMob`` methods.""" @@ -612,30 +603,26 @@ def to_dict(self, *, with_mob_ids: bool = False) -> dict: } @classmethod - def find_by_id(cls, encounter_id: int, *, path: Path) -> ZoneEncounter | None: + def find_by_id(cls, encounter_id: int, *, schema: str) -> ZoneEncounter | None: """Single-encounter fetch by id, with mobs. None if not found.""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: row = conn.execute(_SQL["select_encounter_by_id"], (encounter_id,)).fetchone() return cls._from_row(conn, row) if row else None @classmethod - def _list_for_zone_id(cls, conn: sqlite3.Connection, zone_id: int) -> list[ZoneEncounter]: + def _list_for_zone_id(cls, conn: Any, zone_id: int) -> list[ZoneEncounter]: """List all encounters for a zone using an open connection. Internal helper for ``list_for_zone_name`` and ``_hydrate_zone`` — they both - already hold a connection and shouldn't open a fresh one per call. - ``conn.row_factory`` must be ``sqlite3.Row`` (the caller's - responsibility — every call site sets it).""" + already hold a connection and shouldn't open a fresh one per call.""" return [cls._from_row(conn, r) for r in conn.execute(_SQL["list_encounters_for_zone"], (zone_id,)).fetchall()] @classmethod - def list_for_zone_name(cls, zone_name: str, *, path: Path) -> list[ZoneEncounter]: + def list_for_zone_name(cls, zone_name: str, *, schema: str) -> list[ZoneEncounter]: """All raid encounters in a zone, resolving the zone by canonical name OR alias. Empty list if zone unknown or has no encounters.""" - if not path.exists() or not zone_name: + if not zone_name: return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: row = conn.execute(_SQL["select_zone_id_by_name_lower"], (zone_name.lower(),)).fetchone() if row is None: row = conn.execute( @@ -647,14 +634,15 @@ def list_for_zone_name(cls, zone_name: str, *, path: Path) -> list[ZoneEncounter return cls._list_for_zone_id(conn, row["id"]) @classmethod - def replace_all_for_zone(cls, conn: sqlite3.Connection, zone_id: int, encounters: list[dict]) -> int: + def replace_all_for_zone(cls, conn: Any, zone_id: int, encounters: list[dict]) -> int: """Bulk-replace every encounter (and its mobs) in a zone. Atomic per zone. Re-runnable: wipes both child tables so removed encounters and removed group mobs disappear cleanly. Returns the number of *encounters* written (not individual mobs). - Takes an open ``conn`` rather than a path because the build script - wraps multiple zones in a single outer transaction. + Takes an open ``conn`` rather than opening one because the build + script wraps multiple zones in a single outer transaction (the + caller commits). Each input dict shape:: @@ -684,7 +672,7 @@ def replace_all_for_zone(cls, conn: sqlite3.Connection, zone_id: int, encounters ), ) _row = cur.fetchone() - encounter_id = int(_row[0]) if _row else 0 + encounter_id = int(_row["id"]) if _row else 0 mobs = enc.get("mobs") or [] if not mobs: # Defensive: an encounter with no listed mobs gets one mob @@ -714,20 +702,18 @@ def add_to_zone( position: int | None = None, stage: str | None = None, wiki_url: str | None = None, - path: Path, + schema: str, ) -> ZoneEncounter: """Append a new encounter to a zone with a single primary mob at position 0. If ``position`` is None, appends after the current max; if provided, inserts at that slot — caller is responsible for it being free (UNIQUE(zone_id, position) raises otherwise).""" - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") + with _connect(schema) as conn: if position is None: row = conn.execute(_SQL["max_encounter_position_for_zone"], (zone_id,)).fetchone() position = int(row["p"]) cur = conn.execute(_SQL["insert_encounter"], (zone_id, primary_mob, position, stage, wiki_url)) - enc_id = (cur.fetchone() or (None,))[0] + enc_id = PgCatalogue.fetchval(cur) conn.execute(_SQL["insert_encounter_mob_primary"], (enc_id, primary_mob, primary_mob.lower())) conn.commit() row = conn.execute(_SQL["select_encounter_by_id"], (enc_id,)).fetchone() @@ -739,7 +725,7 @@ def update( primary_mob: str | None = None, stage: str | None = _UNSET, # type: ignore[assignment] wiki_url: str | None = _UNSET, # type: ignore[assignment] - path: Path, + schema: str, ) -> ZoneEncounter: """Edit encounter metadata. When ``primary_mob`` is given, also renames the position-0 mob in zone_encounter_mobs and mirrors the @@ -748,9 +734,7 @@ def update( new_name = primary_mob if primary_mob is not None else self.encounter_name new_stage = self.stage if stage is _UNSET else stage new_wiki = self.wiki_url if wiki_url is _UNSET else wiki_url - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") + with _connect(schema) as conn: conn.execute(_SQL["update_encounter_meta"], (new_name, new_stage, new_wiki, self.id)) if primary_mob is not None: conn.execute( @@ -761,18 +745,16 @@ def update( row = conn.execute(_SQL["select_encounter_by_id"], (self.id,)).fetchone() result = ZoneEncounter._from_row(conn, row) if primary_mob is not None: - _mirror_primary_rename_in_raids_db(self.zone_id, self.encounter_name, primary_mob, path) + _mirror_primary_rename_in_raids_db(self.zone_id, self.encounter_name, primary_mob, schema) return result - def delete(self, *, path: Path) -> bool: + def delete(self, *, schema: str) -> bool: """Delete this encounter. Cascades zone_encounter_mobs via FK and the matching raids_db row (which itself cascades triggers / timers / strategies). Returns True if a row was deleted.""" - zone_name, _exp = _zone_name_and_expansion(self.zone_id, path) - with sqlite3.connect(path) as conn: - conn.execute("PRAGMA foreign_keys = ON;") + zone_name, _exp = _zone_name_and_expansion(self.zone_id, schema) + with _connect(schema) as conn: cur = conn.execute(_SQL["delete_encounter_by_id"], (self.id,)) - conn.commit() if cur.rowcount == 0: return False if zone_name is not None: @@ -784,7 +766,7 @@ def delete(self, *, path: Path) -> bool: return True @staticmethod - def reorder_in_zone(zone_id: int, ordered_ids: list[int], *, path: Path) -> None: + def reorder_in_zone(zone_id: int, ordered_ids: list[int], *, schema: str) -> None: """Atomically renumber the zone's encounters to 1..N matching the given order. ``ordered_ids`` MUST be a complete permutation of the zone's current encounter ids — raises ValueError otherwise. The @@ -793,9 +775,7 @@ def reorder_in_zone(zone_id: int, ordered_ids: list[int], *, path: Path) -> None collisions. Mirrors the new positions onto raids_db rows.""" if len(ordered_ids) != len(set(ordered_ids)): raise ValueError("ordered_ids contains duplicates") - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON;") + with _connect(schema) as conn: current = { r["id"]: (r["encounter_name"], r["position"]) for r in conn.execute(_SQL["list_zone_encounter_positions"], (zone_id,)) @@ -809,14 +789,13 @@ def reorder_in_zone(zone_id: int, ordered_ids: list[int], *, path: Path) -> None ) zone_row = conn.execute(_SQL["select_zone_name_by_id"], (zone_id,)).fetchone() zone_name = zone_row["name"] if zone_row else None - with conn: # single transaction - # Two-phase write to dodge the UNIQUE(zone_id, position) - # collision on mid-update overlap: negative sentinels first, - # then 1..N. - for tmp_neg, enc_id in enumerate(ordered_ids, start=1): - conn.execute(_SQL["update_encounter_position"], (-tmp_neg, enc_id)) - for new_pos, enc_id in enumerate(ordered_ids, start=1): - conn.execute(_SQL["update_encounter_position"], (new_pos, enc_id)) + # Two-phase write to dodge the UNIQUE(zone_id, position) + # collision on mid-update overlap: negative sentinels first, + # then 1..N. Single transaction — the `with` commits on exit. + for tmp_neg, enc_id in enumerate(ordered_ids, start=1): + conn.execute(_SQL["update_encounter_position"], (-tmp_neg, enc_id)) + for new_pos, enc_id in enumerate(ordered_ids, start=1): + conn.execute(_SQL["update_encounter_position"], (new_pos, enc_id)) if zone_name is None: return # Mirror onto raids_db: for each encounter whose primary mob has a @@ -837,8 +816,8 @@ def reorder_in_zone(zone_id: int, ordered_ids: list[int], *, path: Path) -> None # Active-record CRUD for the /raids page admin curation: which expansions # are surfaced, which raid zones within each, and which named "lane" # (category) each zone is dragged into. The featured-raid trio is the only -# part of zones.db the maintainer actively edits via the admin UI; every -# other table is rebuilt from the curated JSON source. +# part of the zones schema the maintainer actively edits via the admin UI; +# every other table is rebuilt from the curated JSON source. # # `_dedup_expansion_rows` factors a quirk: the JOIN-based listing queries # can return multiple rows per (expansion_short) when the same expansion @@ -847,7 +826,7 @@ def reorder_in_zone(zone_id: int, ordered_ids: list[int], *, path: Path) -> None # courtesy of ORDER BY in the SQL). -def _dedup_expansion_rows(rows: list[sqlite3.Row]) -> list[FeaturedRaidExpansion]: +def _dedup_expansion_rows(rows: list[Any]) -> list[FeaturedRaidExpansion]: seen: set[str] = set() result: list[FeaturedRaidExpansion] = [] for r in rows: @@ -876,44 +855,37 @@ def to_dict(self) -> dict: return {"short": self.expansion_short, "name": self.name, "year": self.year} @classmethod - def list_active(cls, *, path: Path) -> list[FeaturedRaidExpansion]: + def list_active(cls, *, schema: str) -> list[FeaturedRaidExpansion]: """Featured expansions (explicit + implicit-via-zones), newest first.""" - if not path.exists(): - return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: rows = conn.execute(_SQL["list_featured_raid_expansions"]).fetchall() return _dedup_expansion_rows(rows) @classmethod - def list_available(cls, *, path: Path) -> list[FeaturedRaidExpansion]: - """Expansions in zones.db NOT yet featured (neither explicit row nor - any zone of theirs featured). The admin 'Add expansion' picker.""" - if not path.exists(): - return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + def list_available(cls, *, schema: str) -> list[FeaturedRaidExpansion]: + """Expansions in the zones schema NOT yet featured (neither explicit + row nor any zone of theirs featured). The admin 'Add expansion' + picker.""" + with _connect(schema) as conn: rows = conn.execute(_SQL["list_available_raid_expansions"]).fetchall() return _dedup_expansion_rows(rows) @classmethod - def create(cls, expansion_short: str, *, path: Path) -> FeaturedRaidExpansion | None: + def create(cls, expansion_short: str, *, schema: str) -> FeaturedRaidExpansion | None: """Mark an expansion as featured. Validates that the expansion is - known to zones.db (returns None otherwise — route layer maps to - 404). Idempotent for already-featured expansions: returns the + known to the zones schema (returns None otherwise — route layer maps + to 404). Idempotent for already-featured expansions: returns the same instance both on fresh-insert and on already-featured.""" - if not path.exists() or not expansion_short: + if not expansion_short: return None - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: meta = conn.execute(_SQL["select_expansion_name_year"], (expansion_short,)).fetchone() if meta is None: return None conn.execute(_SQL["insert_featured_raid_expansion"], (expansion_short,)) - conn.commit() return cls(expansion_short=expansion_short, name=meta["name"], year=meta["year"]) - def remove(self, *, path: Path) -> bool: + def remove(self, *, schema: str) -> bool: """Remove from featured AND cascade-remove this expansion's featured raid zones. Preserves the underlying zone_encounters data — just hides everything from /raids until re-added. @@ -921,19 +893,16 @@ def remove(self, *, path: Path) -> bool: Returns True if the featured_raid_expansions row was removed, False if there was nothing to remove. Cascaded zone deletions don't influence the return value.""" - if not path.exists(): - return False - with sqlite3.connect(path) as conn: - with conn: - conn.execute( - _SQL["remove_featured_raid_zones_in_expansion"], - (self.expansion_short,), - ) - cur = conn.execute( - _SQL["delete_featured_raid_expansion"], - (self.expansion_short,), - ) - return cur.rowcount > 0 + with _connect(schema) as conn: + conn.execute( + _SQL["remove_featured_raid_zones_in_expansion"], + (self.expansion_short,), + ) + cur = conn.execute( + _SQL["delete_featured_raid_expansion"], + (self.expansion_short,), + ) + return cur.rowcount > 0 @dataclass(frozen=True) @@ -955,14 +924,11 @@ class FeaturedRaidZone: category: str | None @classmethod - def list_for_expansion(cls, expansion_short: str, *, path: Path) -> list[FeaturedRaidZone]: + def list_for_expansion(cls, expansion_short: str, *, schema: str) -> list[FeaturedRaidZone]: """All featured raid zones for an expansion, sorted by - (category, position). NULL categories sort first (SQLite default), - which lands the implicit Uncategorised lane at the top.""" - if not path.exists(): - return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + (category, position). NULL categories sort first (NULLS FIRST in + the SQL), which lands the implicit Uncategorised lane at the top.""" + with _connect(schema) as conn: rows = conn.execute( _SQL["list_featured_raid_zones"].format(cols=_SELECT_COLS), (expansion_short,), @@ -979,7 +945,7 @@ def list_for_expansion(cls, expansion_short: str, *, path: Path) -> list[Feature ] @classmethod - def add(cls, zone_name: str, *, path: Path) -> FeaturedRaidZone | None: + def add(cls, zone_name: str, *, schema: str) -> FeaturedRaidZone | None: """Mark a raid zone as featured. Validates that the zone exists AND is tagged raid_x4 or raid_x2 — we don't want random zones surfacing on /raids just because admin typed a name. Lands in @@ -987,10 +953,9 @@ def add(cls, zone_name: str, *, path: Path) -> FeaturedRaidZone | None: new entry shows at the bottom; admin drags into a named lane afterwards. Returns the new instance, or None on validation failure (route maps to 400).""" - if not path.exists() or not zone_name: + if not zone_name: return None - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: zone = conn.execute( _SQL["find_zone_by_name_lower"].format(cols=_SELECT_COLS), (zone_name.lower(),), @@ -1004,12 +969,11 @@ def add(cls, zone_name: str, *, path: Path) -> FeaturedRaidZone | None: _SQL["max_featured_position_uncategorised"], (zone["expansion_short"],), ).fetchone() - new_position = (max_pos_row[0] if max_pos_row else -1) + 1 + new_position = (max_pos_row["p"] if max_pos_row else -1) + 1 conn.execute( _SQL["insert_featured_raid_zone_uncategorised"], (zone["id"], new_position), ) - conn.commit() return cls( zone_id=zone["id"], zone_name=zone["name"], @@ -1018,21 +982,18 @@ def add(cls, zone_name: str, *, path: Path) -> FeaturedRaidZone | None: category=None, ) - def remove(self, *, path: Path) -> bool: + def remove(self, *, schema: str) -> bool: """Remove from featured. Preserves zone_encounters boss data so re-adding restores the lane. Returns True if a row was removed.""" - if not path.exists(): - return False - with sqlite3.connect(path) as conn: + with _connect(schema) as conn: cur = conn.execute( _SQL["delete_featured_raid_zone_by_name"], (self.zone_name.lower(),), ) - conn.commit() return cur.rowcount > 0 @staticmethod - def reorder_in_expansion(expansion_short: str, ordering: list[dict], *, path: Path) -> bool: + def reorder_in_expansion(expansion_short: str, ordering: list[dict], *, schema: str) -> bool: """Atomically rewrite category + position for every zone in ``ordering``. Each entry: ``{"name": str, "category": str | None, "position": int}``. @@ -1048,47 +1009,44 @@ def reorder_in_expansion(expansion_short: str, ordering: list[dict], *, path: Pa Returns False if any zone in ``ordering`` isn't currently featured in this expansion (route layer maps to 400).""" - if not path.exists(): - return False - with sqlite3.connect(path) as conn: - with conn: - # Validate every zone — surfaces typos / stale clients early. - zone_ids: dict[str, int] = {} - for entry in ordering: - row = conn.execute( - _SQL["find_featured_zone_id_in_expansion"], - (entry["name"].lower(), expansion_short), - ).fetchone() - if not row: - return False - zone_ids[entry["name"]] = row[0] - # Auto-create missing categories at end of the existing run. - seen_categories = {e["category"] for e in ordering if e.get("category")} - if seen_categories: - max_pos_row = conn.execute( - _SQL["max_category_position"], - (expansion_short,), - ).fetchone() - next_pos = (max_pos_row[0] if max_pos_row else -1) + 1 - for cat in seen_categories: - cur = conn.execute( - _SQL["insert_featured_raid_category_or_ignore"], - (expansion_short, cat, next_pos), - ) - if cur.rowcount: - next_pos += 1 - # Two-phase write. - for i, entry in enumerate(ordering): - conn.execute( - _SQL["update_featured_raid_zone_position_and_category"], - (-(i + 1), entry.get("category"), zone_ids[entry["name"]]), - ) - for entry in ordering: - conn.execute( - _SQL["update_featured_raid_zone_position"], - (entry["position"], zone_ids[entry["name"]]), + with _connect(schema) as conn: + # Validate every zone — surfaces typos / stale clients early. + zone_ids: dict[str, int] = {} + for entry in ordering: + row = conn.execute( + _SQL["find_featured_zone_id_in_expansion"], + (entry["name"].lower(), expansion_short), + ).fetchone() + if not row: + return False + zone_ids[entry["name"]] = row["id"] + # Auto-create missing categories at end of the existing run. + seen_categories = {e["category"] for e in ordering if e.get("category")} + if seen_categories: + max_pos_row = conn.execute( + _SQL["max_category_position"], + (expansion_short,), + ).fetchone() + next_pos = (max_pos_row["p"] if max_pos_row else -1) + 1 + for cat in seen_categories: + cur = conn.execute( + _SQL["insert_featured_raid_category_or_ignore"], + (expansion_short, cat, next_pos), ) - return True + if cur.rowcount: + next_pos += 1 + # Two-phase write — one transaction, committed on `with` exit. + for i, entry in enumerate(ordering): + conn.execute( + _SQL["update_featured_raid_zone_position_and_category"], + (-(i + 1), entry.get("category"), zone_ids[entry["name"]]), + ) + for entry in ordering: + conn.execute( + _SQL["update_featured_raid_zone_position"], + (entry["position"], zone_ids[entry["name"]]), + ) + return True @dataclass(frozen=True) @@ -1109,12 +1067,9 @@ def to_dict(self) -> dict: return {"name": self.name, "position": self.position} @classmethod - def list_for_expansion(cls, expansion_short: str, *, path: Path) -> list[FeaturedRaidCategory]: + def list_for_expansion(cls, expansion_short: str, *, schema: str) -> list[FeaturedRaidCategory]: """Admin-defined categories in saved order.""" - if not path.exists(): - return [] - with sqlite3.connect(path) as conn: - conn.row_factory = sqlite3.Row + with _connect(schema) as conn: rows = conn.execute( _SQL["list_featured_raid_categories"], (expansion_short,), @@ -1122,12 +1077,10 @@ def list_for_expansion(cls, expansion_short: str, *, path: Path) -> list[Feature return [cls(expansion_short=expansion_short, name=r["name"], position=r["position"]) for r in rows] @classmethod - def create(cls, expansion_short: str, name: str, *, path: Path) -> FeaturedRaidCategory | None: + def create(cls, expansion_short: str, name: str, *, schema: str) -> FeaturedRaidCategory | None: """Create an empty category lane at MAX+1 position. Returns None if a category by this name already exists for the expansion.""" - if not path.exists(): - return None - with sqlite3.connect(path) as conn: + with _connect(schema) as conn: existing = conn.execute( _SQL["check_featured_raid_category_exists"], (expansion_short, name), @@ -1138,62 +1091,55 @@ def create(cls, expansion_short: str, name: str, *, path: Path) -> FeaturedRaidC _SQL["max_category_position"], (expansion_short,), ).fetchone() - new_pos = (max_pos[0] if max_pos else -1) + 1 + new_pos = (max_pos["p"] if max_pos else -1) + 1 conn.execute( _SQL["insert_featured_raid_category"], (expansion_short, name, new_pos), ) - conn.commit() return cls(expansion_short=expansion_short, name=name, position=new_pos) - def delete(self, *, path: Path) -> bool: + def delete(self, *, schema: str) -> bool: """Delete this category. Zones currently in it have their category set to NULL (move to Uncategorised). Returns True if a row was deleted.""" - if not path.exists(): - return False - with sqlite3.connect(path) as conn: - with conn: - conn.execute( - _SQL["move_featured_zones_to_null_category"], - (self.name, self.expansion_short), - ) - cur = conn.execute( - _SQL["delete_featured_raid_category"], - (self.expansion_short, self.name), - ) - return cur.rowcount > 0 + with _connect(schema) as conn: + conn.execute( + _SQL["move_featured_zones_to_null_category"], + (self.name, self.expansion_short), + ) + cur = conn.execute( + _SQL["delete_featured_raid_category"], + (self.expansion_short, self.name), + ) + return cur.rowcount > 0 @staticmethod - def reorder_in_expansion(expansion_short: str, ordering: list[dict], *, path: Path) -> bool: + def reorder_in_expansion(expansion_short: str, ordering: list[dict], *, schema: str) -> bool: """Atomic two-phase position rewrite for category lanes. Each entry: ``{"name": str, "position": int}``. Returns False if any name in ``ordering`` isn't a category in this expansion (route layer maps to 400).""" - if not path.exists(): - return False - with sqlite3.connect(path) as conn: - with conn: - for entry in ordering: - row = conn.execute( - _SQL["check_featured_raid_category_exists"], - (expansion_short, entry["name"]), - ).fetchone() - if not row: - return False - # Two-phase write: temp negatives, then final positions. - for i, entry in enumerate(ordering): - conn.execute( - _SQL["update_featured_raid_category_position"], - (-(i + 1), expansion_short, entry["name"]), - ) - for entry in ordering: - conn.execute( - _SQL["update_featured_raid_category_position"], - (entry["position"], expansion_short, entry["name"]), - ) - return True + with _connect(schema) as conn: + for entry in ordering: + row = conn.execute( + _SQL["check_featured_raid_category_exists"], + (expansion_short, entry["name"]), + ).fetchone() + if not row: + return False + # Two-phase write: temp negatives, then final positions. + for i, entry in enumerate(ordering): + conn.execute( + _SQL["update_featured_raid_category_position"], + (-(i + 1), expansion_short, entry["name"]), + ) + for entry in ordering: + conn.execute( + _SQL["update_featured_raid_category_position"], + (entry["position"], expansion_short, entry["name"]), + ) + return True # --------------------------------------------------------------------------- @@ -1201,73 +1147,31 @@ def reorder_in_expansion(expansion_short: str, ordering: list[dict], *, path: Pa # --------------------------------------------------------------------------- -class ZoneCatalogue(BaseCatalogue): - """Read (and build) access to one zones.db file. +class ZoneCatalogue(PgCatalogue): + """Read (and build) access to one zones schema. - The eq2db data-interface convention (see :class:`BaseCatalogue`): - the DB path lives on the instance; the shared module-level ``catalogue`` - is the runtime entry point, and tests construct ``ZoneCatalogue(tmp_db)``. + The eq2db data-interface convention (see :class:`PgCatalogue`): the + schema name lives on the instance; the shared module-level ``catalogue`` + is the runtime entry point, and tests construct + ``ZoneCatalogue(scratch_schema)``. The frozen dataclass models above (Zone, ZoneEncounter, ZoneEncounterMob, FeaturedRaid*) are the typed active-record layer — the catalogue methods - delegate to them with ``self.path`` and convert to the legacy dict shapes - the routes and scripts consume, so consumers never juggle paths or model - imports themselves. + delegate to them with ``self.schema`` and convert to the legacy dict + shapes the routes and scripts consume, so consumers never juggle + connections or model imports themselves. """ - # ON DELETE CASCADE on the zone_types / zone_aliases child tables - # only fires with the per-connection FK pragma. - FOREIGN_KEYS = True - - def __init__(self, path: Path = DB_PATH) -> None: - super().__init__(path) - - def _create_schema(self, conn: sqlite3.Connection) -> None: - conn.execute(_SQL["schema_zones"]) - conn.execute(_SQL["schema_zone_types"]) - conn.execute(_SQL["schema_zone_aliases"]) - conn.execute(_SQL["schema_zone_encounters"]) - conn.execute(_SQL["schema_zone_encounter_mobs"]) - conn.execute(_SQL["schema_featured_raid_expansions"]) - conn.execute(_SQL["schema_featured_raid_zones"]) - conn.execute(_SQL["schema_featured_raid_categories"]) - # Migration: zone categories + position for drag-reorder. - # Idempotent — already-applied schemas raise OperationalError on the - # duplicate-column attempt, which we swallow. - self._apply_migrations(conn, (_SQL["migrate_add_featured_position"], _SQL["migrate_add_featured_category"])) - # Migration: drop the pre-v2 zone_bosses table if it lingers from - # an older DB build. No need to preserve data — bosses are always - # rebuilt from the curated source file. - conn.execute(_SQL["drop_legacy_zone_bosses"]) - conn.executescript(_SQL["indexes_all"]) - # One-time data normalization (idempotent): legacy `encounter_name` - # values were the comma-joined display of every mob in the encounter - # ("Ire, Malevolence"). The web roster editor treats encounter_name - # as the PRIMARY mob's name (kept in sync with the mob at - # position 0). Rewrite any comma-containing row to its position-0 - # mob name; rows without any mobs are left untouched. - # NOTE: Not version-gated — this UPDATE is cheap (only touches rows - # with commas in the name) and must remain idempotent across multiple - # init_db calls (see test_init_db_normalizes_comma_joined_encounter_name). - conn.execute(_SQL["normalise_comma_joined_encounter_names"]) - # One-time data normalization (idempotent): strip the wiki-import - # " (Zone)" disambiguator suffix from zone names (e.g. "Kurn's Tower - # (Zone)" → "Kurn's Tower"). EQ2i uses the parenthetical to - # disambiguate a wiki article from the in-game zone of the same name; - # the in-game logs and our UI both use the bare name. The old name - # is also inserted as an alias so anything historically referencing - # the parenthesised form still resolves via find_by_name. Idempotent: - # subsequent runs match zero rows (LIKE filter no longer hits the - # already-cleaned names). - conn.execute(_SQL["normalise_paren_zone_to_alias"]) - conn.execute(_SQL["normalise_strip_paren_zone"]) + def __init__(self, schema: str = SCHEMA) -> None: + super().__init__(schema) # ── Build (scripts/build_zones_db.py) ──────────────────────────────────── @staticmethod def zone_to_row(z: dict) -> dict: """Flatten a cleaned-JSON zone record into the columns of the zones - table. `types` and `aliases` are handled separately by upsert_zones.""" + table. `types` and `aliases` are handled separately by upsert_zones, + which also assigns the builder-owned ``id``.""" name = z["name"] cls = z["classification"] exp = cls["expansion"] @@ -1295,44 +1199,72 @@ def zone_to_row(z: dict) -> dict: "wiki_url": (z.get("source_pages") or [None])[0], } - def upsert_zones(self, zones: list[dict], conn: sqlite3.Connection) -> int: + def upsert_zones(self, zones: list[dict], conn: Any) -> int: """Bulk upsert from cleaned-JSON zone records. + ``zones.id`` is builder-assigned (plain bigint PK, no identity): + each source name is resolved to its existing id — or allocated the + next free id — BEFORE the upsert, and the upsert conflicts on id. + A rebuild therefore never changes an existing zone's id, which is + what keeps the curator-managed ``zone_encounters`` rows (FK → + zones.id, ON DELETE CASCADE) attached across metadata rebuilds. + For each input zone: - * Insert/replace the row in `zones`. + * Insert/update the row in `zones` (id-preserving upsert). * Replace its rows in `zone_types` (so removed types disappear). * Replace its rows in `zone_aliases` (so removed aliases disappear). - Atomic per-zone within a single transaction. Re-runnable. + Commits once at the end (single transaction for the whole batch). + Re-runnable. Zones REMOVED from the source are NOT touched here — + see :meth:`delete_zones_not_named`, which the build script calls + explicitly so the prune count is visible in its report. """ + existing: dict[str, int] = {r["name"]: r["id"] for r in conn.execute(_SQL["select_all_zone_ids"])} + next_id = (max(existing.values()) + 1) if existing else 1 n = 0 - with conn: # single transaction for the whole batch - for z in zones: - row = self.zone_to_row(z) - conn.execute(_SQL["upsert_zone"], row) - zone_id = conn.execute(_SQL["select_zone_id_by_name"], (z["name"],)).fetchone()[0] - - # Reset and repopulate types + aliases for this zone. Cheaper - # than diffing on every rebuild. - conn.execute(_SQL["delete_zone_types_for_zone"], (zone_id,)) - types = z["classification"].get("types") or [] - if types: - conn.executemany( - _SQL["insert_zone_type"], - [(zone_id, t) for t in types], - ) + for z in zones: + row = self.zone_to_row(z) + zone_id = existing.get(row["name"]) + if zone_id is None: + zone_id = next_id + next_id += 1 + existing[row["name"]] = zone_id + row["id"] = zone_id + conn.execute(_SQL["upsert_zone"], row) + + # Reset and repopulate types + aliases for this zone. Cheaper + # than diffing on every rebuild. + conn.execute(_SQL["delete_zone_types_for_zone"], (zone_id,)) + types = z["classification"].get("types") or [] + if types: + conn.executemany( + _SQL["insert_zone_type"], + [(zone_id, t) for t in types], + ) - conn.execute(_SQL["delete_zone_aliases_for_zone"], (zone_id,)) - aliases = z.get("aliases") or [] - if aliases: - conn.executemany( - _SQL["insert_zone_alias"], - [(a, a.lower(), zone_id) for a in aliases], - ) - n += 1 + conn.execute(_SQL["delete_zone_aliases_for_zone"], (zone_id,)) + aliases = z.get("aliases") or [] + if aliases: + conn.executemany( + _SQL["insert_zone_alias"], + [(a, a.lower(), zone_id) for a in aliases], + ) + n += 1 + conn.commit() return n - def replace_bosses_for_zone(self, conn: sqlite3.Connection, zone_id: int, encounters: list[dict]) -> int: + @staticmethod + def delete_zones_not_named(conn: Any, names: list[str]) -> int: + """Prune zones whose canonical name no longer appears in the source + JSON. CASCADE removes their types/aliases AND any curator-entered + encounters — which is correct for a genuinely-removed zone, but the + build script prints the count so an unexpected prune is visible. + Commits. Returns the number of zones removed.""" + cur = conn.execute(_SQL["delete_zones_not_named"], (list(names),)) + conn.commit() + return cur.rowcount + + def replace_bosses_for_zone(self, conn: Any, zone_id: int, encounters: list[dict]) -> int: """Replace the encounters list for a zone. Atomic per-zone. Delegates to ``ZoneEncounter.replace_all_for_zone`` (takes an open conn because the build script wraps multiple zones in one @@ -1344,51 +1276,61 @@ def replace_bosses_for_zone(self, conn: sqlite3.Connection, zone_id: int, encoun def find_by_name(self, name: str) -> dict | None: """Resolve a zone by name (canonical → alias fallback). Returns the legacy hydrated dict, or None on miss.""" - z = Zone.find_by_name(name, path=self.path) + z = Zone.find_by_name(name, schema=self.schema) return z.to_dict() if z is not None else None def list_by_expansion(self, short: str, type_filter: str | None = None) -> list[dict]: """All zones in an expansion as legacy hydrated dicts. ``type_filter`` stays positional for the existing call sites; the model exposes it as a keyword-only argument.""" - return [z.to_dict() for z in Zone.list_by_expansion(short, type_filter=type_filter, path=self.path)] + return [z.to_dict() for z in Zone.list_by_expansion(short, type_filter=type_filter, schema=self.schema)] def list_by_event(self, event_name: str) -> list[dict]: - return [z.to_dict() for z in Zone.list_by_event(event_name, path=self.path)] + return [z.to_dict() for z in Zone.list_by_event(event_name, schema=self.schema)] def list_by_type(self, type_token: str) -> list[dict]: - return [z.to_dict() for z in Zone.list_by_type(type_token, path=self.path)] + return [z.to_dict() for z in Zone.list_by_type(type_token, schema=self.schema)] def list_bosses_for_zone(self, zone_name: str) -> list[dict]: """All raid encounters in a zone (looked up by canonical name OR alias). Mob ids included — the editor frontend targets individual mobs.""" - return [enc.to_dict(with_mob_ids=True) for enc in ZoneEncounter.list_for_zone_name(zone_name, path=self.path)] + return [ + enc.to_dict(with_mob_ids=True) for enc in ZoneEncounter.list_for_zone_name(zone_name, schema=self.schema) + ] def find_zones_by_boss(self, mob_name: str) -> list[dict]: """Reverse lookup: which zone(s) host a given raid boss? See ``Zone.find_by_boss``.""" - return [z.to_dict() for z in Zone.find_by_boss(mob_name, path=self.path)] + return [z.to_dict() for z in Zone.find_by_boss(mob_name, schema=self.schema)] def list_expansions(self) -> list[dict]: """Return distinct expansions ordered newest first (by expansion_year DESC). Each entry is ``{"short": expansion_short, "name": expansion_name}``. - Returns [] when zones.db is missing or the zones table does not yet exist - (graceful degradation — the admin endpoint must never 500 on a missing DB). """ - rows = self._fetchall(_SQL["list_distinct_expansions"]) + conn = self.init_db() + try: + rows = conn.execute(_SQL["list_distinct_expansions"]).fetchall() + finally: + conn.close() # De-duplicate by short (same short can have multiple rows with the same year). seen: set[str] = set() result: list[dict] = [] - for short, name, _year in rows: + for r in rows: + short = r["expansion_short"] if short not in seen: seen.add(short) - result.append({"short": short, "name": name}) + result.append({"short": short, "name": r["expansion_name"]}) return result def expansion_counts(self) -> dict[str, int]: """Diagnostic: zones per expansion short. Used by the build report.""" - return {r[0]: r[1] for r in self._fetchall(_SQL["expansion_counts"])} + conn = self.init_db() + try: + rows = conn.execute(_SQL["expansion_counts"]).fetchall() + finally: + conn.close() + return {r["expansion_short"]: r["n"] for r in rows} # ── Editable encounter + mob CRUD (used by the zones-admin routes) ─────── @@ -1402,7 +1344,7 @@ def add_encounter( wiki_url: str | None = None, ) -> dict: return ZoneEncounter.add_to_zone( - zone_id, primary_mob=primary_mob, position=position, stage=stage, wiki_url=wiki_url, path=self.path + zone_id, primary_mob=primary_mob, position=position, stage=stage, wiki_url=wiki_url, schema=self.schema ).to_dict() def update_encounter( @@ -1413,45 +1355,45 @@ def update_encounter( stage: str | None = _UNSET, # type: ignore[assignment] wiki_url: str | None = _UNSET, # type: ignore[assignment] ) -> dict: - enc = ZoneEncounter.find_by_id(encounter_id, path=self.path) + enc = ZoneEncounter.find_by_id(encounter_id, schema=self.schema) if enc is None: raise LookupError(f"zone_encounter {encounter_id} not found") - return enc.update(primary_mob=primary_mob, stage=stage, wiki_url=wiki_url, path=self.path).to_dict() + return enc.update(primary_mob=primary_mob, stage=stage, wiki_url=wiki_url, schema=self.schema).to_dict() def reorder_encounters(self, zone_id: int, ordered_encounter_ids: list[int]) -> None: - ZoneEncounter.reorder_in_zone(zone_id, ordered_encounter_ids, path=self.path) + ZoneEncounter.reorder_in_zone(zone_id, ordered_encounter_ids, schema=self.schema) def delete_encounter(self, encounter_id: int) -> bool: - enc = ZoneEncounter.find_by_id(encounter_id, path=self.path) + enc = ZoneEncounter.find_by_id(encounter_id, schema=self.schema) if enc is None: return False - return enc.delete(path=self.path) + return enc.delete(schema=self.schema) def list_mobs(self, encounter_id: int) -> list[dict]: - return [m.to_dict() for m in ZoneEncounterMob.list_for_encounter(encounter_id, path=self.path)] + return [m.to_dict() for m in ZoneEncounterMob.list_for_encounter(encounter_id, schema=self.schema)] def add_mob(self, encounter_id: int, *, mob_name: str, make_primary: bool = False) -> dict: return ZoneEncounterMob.add_to_encounter( - encounter_id, mob_name=mob_name, make_primary=make_primary, path=self.path + encounter_id, mob_name=mob_name, make_primary=make_primary, schema=self.schema ).to_dict() def update_mob(self, mob_id: int, *, mob_name: str) -> dict: - mob = ZoneEncounterMob.find_by_id(mob_id, path=self.path) + mob = ZoneEncounterMob.find_by_id(mob_id, schema=self.schema) if mob is None: raise LookupError(f"zone_encounter_mob {mob_id} not found") - return mob.rename(mob_name, path=self.path).to_dict() + return mob.rename(mob_name, schema=self.schema).to_dict() def promote_mob(self, mob_id: int) -> dict: - mob = ZoneEncounterMob.find_by_id(mob_id, path=self.path) + mob = ZoneEncounterMob.find_by_id(mob_id, schema=self.schema) if mob is None: raise LookupError(f"zone_encounter_mob {mob_id} not found") - return mob.promote_to_primary(path=self.path).to_dict() + return mob.promote_to_primary(schema=self.schema).to_dict() def delete_mob(self, mob_id: int) -> bool: - mob = ZoneEncounterMob.find_by_id(mob_id, path=self.path) + mob = ZoneEncounterMob.find_by_id(mob_id, schema=self.schema) if mob is None: return False - return mob.delete(path=self.path) + return mob.delete(schema=self.schema) # ── Zone-type tag helpers (used by the dungeon-curation UI on /raids) ──── @@ -1459,42 +1401,39 @@ def add_zone_type(self, zone_name: str, type_token: str) -> dict | None: """Add a type tag (e.g. 'dungeon') to a zone. Returns None if the zone_name doesn't resolve; route layer is responsible for turning that into a 404.""" - z = Zone.find_by_name(zone_name, path=self.path) + z = Zone.find_by_name(zone_name, schema=self.schema) if z is None: return None - return z.add_type(type_token, path=self.path).to_dict() + return z.add_type(type_token, schema=self.schema).to_dict() def remove_zone_type(self, zone_name: str, type_token: str) -> dict | None: """Remove a type tag from a zone. See ``Zone.remove_type``.""" - z = Zone.find_by_name(zone_name, path=self.path) + z = Zone.find_by_name(zone_name, schema=self.schema) if z is None: return None - return z.remove_type(type_token, path=self.path).to_dict() + return z.remove_type(type_token, schema=self.schema).to_dict() # ── Featured-raid curation (the /raids page admin UI) ──────────────────── def list_featured_raid_expansions(self) -> list[dict]: - return [e.to_dict() for e in FeaturedRaidExpansion.list_active(path=self.path)] + return [e.to_dict() for e in FeaturedRaidExpansion.list_active(schema=self.schema)] def list_available_raid_expansions(self) -> list[dict]: - return [e.to_dict() for e in FeaturedRaidExpansion.list_available(path=self.path)] + return [e.to_dict() for e in FeaturedRaidExpansion.list_available(schema=self.schema)] def add_featured_raid_expansion(self, expansion_short: str) -> bool: - return FeaturedRaidExpansion.create(expansion_short, path=self.path) is not None + return FeaturedRaidExpansion.create(expansion_short, schema=self.schema) is not None def remove_featured_raid_expansion(self, expansion_short: str) -> bool: # Key-only delete; the model.remove() only needs expansion_short # and the stub instance carries that. - return FeaturedRaidExpansion(expansion_short=expansion_short, name="", year=None).remove(path=self.path) + return FeaturedRaidExpansion(expansion_short=expansion_short, name="", year=None).remove(schema=self.schema) def list_featured_raid_zones(self, expansion_short: str) -> list[dict]: # Routes need the full hydrated zone shape (types/aliases/bosses) # PLUS the featuring metadata (position + category). The model only # carries the featuring metadata, so this joins via _hydrate_zone. - if not self.path.exists(): - return [] - with sqlite3.connect(self.path) as conn: - conn.row_factory = sqlite3.Row + with self.init_db() as conn: rows = conn.execute( _SQL["list_featured_raid_zones"].format(cols=_SELECT_COLS), (expansion_short,), @@ -1512,10 +1451,7 @@ def list_available_raid_zones(self, expansion_short: str) -> list[dict]: # full zone info). Not exposed via the FeaturedRaidZone model because # the returned shape is Zone, not FeaturedRaidZone — they're not yet # featured. - if not self.path.exists(): - return [] - with sqlite3.connect(self.path) as conn: - conn.row_factory = sqlite3.Row + with self.init_db() as conn: rows = conn.execute( _SQL["list_available_raid_zones"].format(cols=_SELECT_COLS), (expansion_short,), @@ -1525,7 +1461,7 @@ def list_available_raid_zones(self, expansion_short: str) -> list[dict]: def add_featured_raid_zone(self, zone_name: str) -> dict | None: # Legacy contract: returns hydrated zone dict (NOT the FeaturedRaidZone # shape). Re-hydrate by name after the model add() succeeds. - fz = FeaturedRaidZone.add(zone_name, path=self.path) + fz = FeaturedRaidZone.add(zone_name, schema=self.schema) if fz is None: return None return self.find_by_name(fz.zone_name) @@ -1534,24 +1470,24 @@ def remove_featured_raid_zone(self, zone_name: str) -> bool: # Stub instance with name only — remove() uses .zone_name as the # delete key; other fields are unused. return FeaturedRaidZone(zone_id=0, zone_name=zone_name, expansion_short="", position=0, category=None).remove( - path=self.path + schema=self.schema ) def reorder_featured_raid_zones(self, expansion_short: str, ordering: list[dict]) -> bool: - return FeaturedRaidZone.reorder_in_expansion(expansion_short, ordering, path=self.path) + return FeaturedRaidZone.reorder_in_expansion(expansion_short, ordering, schema=self.schema) def list_featured_raid_categories(self, expansion_short: str) -> list[dict]: - return [c.to_dict() for c in FeaturedRaidCategory.list_for_expansion(expansion_short, path=self.path)] + return [c.to_dict() for c in FeaturedRaidCategory.list_for_expansion(expansion_short, schema=self.schema)] def create_featured_raid_category(self, expansion_short: str, name: str) -> bool: - return FeaturedRaidCategory.create(expansion_short, name, path=self.path) is not None + return FeaturedRaidCategory.create(expansion_short, name, schema=self.schema) is not None def delete_featured_raid_category(self, expansion_short: str, name: str) -> bool: # Stub instance — delete() only needs expansion_short + name as keys. - return FeaturedRaidCategory(expansion_short=expansion_short, name=name, position=0).delete(path=self.path) + return FeaturedRaidCategory(expansion_short=expansion_short, name=name, position=0).delete(schema=self.schema) def reorder_featured_raid_categories(self, expansion_short: str, ordering: list[dict]) -> bool: - return FeaturedRaidCategory.reorder_in_expansion(expansion_short, ordering, path=self.path) + return FeaturedRaidCategory.reorder_in_expansion(expansion_short, ordering, schema=self.schema) # The shared default instance — every runtime consumer goes through this. diff --git a/backend/eq2db/zones.sql b/backend/eq2db/zones.sql index d31e6d71..d6563a8a 100644 --- a/backend/eq2db/zones.sql +++ b/backend/eq2db/zones.sql @@ -1,161 +1,22 @@ --- SQL for backend/eq2db/zones.py. Schema + DML. The shared `_meta` table --- is created from backend/eq2db/_meta.sql via _meta.create_table(). +-- SQL for backend/eq2db/zones.py — DML against the Postgres `zones` schema. +-- Schema DDL lives in db/migrations/0004_zones.sql (migrations own DDL; the +-- catalogue never creates tables). psycopg %s / %(name)s placeholders. +-- +-- Blocks containing the `{cols}` format field are composed with +-- `.format(cols=_SELECT_COLS)` at call time — keep their bodies brace-free +-- and comment-free so str.format can't trip on stray `{`/`}`. -- --------------------------------------------------------------------------- --- Schema +-- _meta provenance (zones-schema copy of the shared eq2db helper; the shared +-- backend/eq2db/_meta.py stays ?-dialect for the SQLite catalogues) -- --------------------------------------------------------------------------- --- :name schema_zones -CREATE TABLE IF NOT EXISTS zones ( - -- Identity - id INTEGER PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - name_lower TEXT NOT NULL, - - -- Expansion attribution - expansion_short TEXT NOT NULL, -- 'DoF', 'AoM', 'CoE', ... - expansion_name TEXT NOT NULL, -- 'Desert of Flames', ... - expansion_year INTEGER, - expansion_confidence TEXT NOT NULL, -- 'category', 'live_update', ... - expansion_source TEXT, -- audit trail / reason - - -- Flags - is_persistent_instance INTEGER NOT NULL DEFAULT 0, - is_endless_persistent INTEGER NOT NULL DEFAULT 0, - is_tradeskill INTEGER NOT NULL DEFAULT 0, - is_pvp INTEGER NOT NULL DEFAULT 0, - is_openworld INTEGER NOT NULL DEFAULT 0, - is_instance INTEGER NOT NULL DEFAULT 0, - is_live_event INTEGER NOT NULL DEFAULT 0, - is_city INTEGER NOT NULL DEFAULT 0, - is_contested INTEGER NOT NULL DEFAULT 0, - is_deprecated INTEGER NOT NULL DEFAULT 0, - - -- Optional metadata - event_name TEXT, -- when is_live_event=1 - wiki_url TEXT -); - --- Many-to-many zone ↔ type. A zone with both Solo and Group variants gets --- two rows so a "all group zones in RoK" query is one indexed JOIN. --- :name schema_zone_types -CREATE TABLE IF NOT EXISTS zone_types ( - zone_id INTEGER NOT NULL REFERENCES zones(id) ON DELETE CASCADE, - type TEXT NOT NULL, - PRIMARY KEY (zone_id, type) -); - --- Alias → canonical zone. ACT logs may emit "The Fabled Deathtoll" or --- "Fabled Deathtoll"; find_by_name checks aliases before failing. --- :name schema_zone_aliases -CREATE TABLE IF NOT EXISTS zone_aliases ( - alias TEXT NOT NULL PRIMARY KEY, - alias_lower TEXT NOT NULL, - zone_id INTEGER NOT NULL REFERENCES zones(id) ON DELETE CASCADE -); - --- Raid encounters per zone — hand-curated. Each row is a named encounter --- (1 mob solo, or 2-4 mobs grouped). encounter_name is the display label; --- individual mob names live in zone_encounter_mobs for reverse lookup. --- :name schema_zone_encounters -CREATE TABLE IF NOT EXISTS zone_encounters ( - id INTEGER PRIMARY KEY, - zone_id INTEGER NOT NULL REFERENCES zones(id) ON DELETE CASCADE, - encounter_name TEXT NOT NULL, - position INTEGER NOT NULL, - stage TEXT, - wiki_url TEXT, - UNIQUE (zone_id, position) -); - --- :name schema_zone_encounter_mobs -CREATE TABLE IF NOT EXISTS zone_encounter_mobs ( - id INTEGER PRIMARY KEY, - encounter_id INTEGER NOT NULL REFERENCES zone_encounters(id) ON DELETE CASCADE, - mob_name TEXT NOT NULL, - mob_name_lower TEXT NOT NULL, - position INTEGER NOT NULL DEFAULT 0 -); - --- :name schema_featured_raid_expansions -CREATE TABLE IF NOT EXISTS featured_raid_expansions ( - expansion_short TEXT PRIMARY KEY, - added_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) -); - --- :name schema_featured_raid_zones -CREATE TABLE IF NOT EXISTS featured_raid_zones ( - zone_id INTEGER PRIMARY KEY REFERENCES zones(id) ON DELETE CASCADE, - added_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) -); - --- :name schema_featured_raid_categories -CREATE TABLE IF NOT EXISTS featured_raid_categories ( - expansion_short TEXT NOT NULL, - name TEXT NOT NULL, - position INTEGER NOT NULL, - PRIMARY KEY (expansion_short, name) -); - --- :name indexes_all -CREATE INDEX IF NOT EXISTS idx_zones_name_lower ON zones (name_lower); -CREATE INDEX IF NOT EXISTS idx_zones_expansion ON zones (expansion_short); -CREATE INDEX IF NOT EXISTS idx_zones_event ON zones (is_live_event, event_name); -CREATE INDEX IF NOT EXISTS idx_zones_tradeskill ON zones (is_tradeskill); -CREATE INDEX IF NOT EXISTS idx_zone_types_type ON zone_types (type); -CREATE INDEX IF NOT EXISTS idx_zone_types_zone ON zone_types (zone_id); -CREATE INDEX IF NOT EXISTS idx_zone_aliases_lower ON zone_aliases (alias_lower); -CREATE INDEX IF NOT EXISTS idx_zone_aliases_zone ON zone_aliases (zone_id); -CREATE INDEX IF NOT EXISTS idx_zone_enc_zone ON zone_encounters (zone_id, position); -CREATE INDEX IF NOT EXISTS idx_zone_enc_mobs_enc ON zone_encounter_mobs (encounter_id, position); -CREATE INDEX IF NOT EXISTS idx_zone_enc_mobs_lower ON zone_encounter_mobs (mob_name_lower); +-- :name meta_select +SELECT value FROM _meta WHERE key = %s; --- --------------------------------------------------------------------------- --- Idempotent migrations + one-time normalisations --- --------------------------------------------------------------------------- - --- :name migrate_add_featured_position -ALTER TABLE featured_raid_zones ADD COLUMN position INTEGER NOT NULL DEFAULT 0; - --- :name migrate_add_featured_category -ALTER TABLE featured_raid_zones ADD COLUMN category TEXT; - --- :name drop_legacy_zone_bosses -DROP TABLE IF EXISTS zone_bosses; - --- Legacy encounter_name values were the comma-joined display of every mob --- ("Ire, Malevolence"). The roster editor treats encounter_name as the --- primary mob's name. Rewrite any comma-containing row to its position-0 --- mob name. Idempotent: rows without commas don't match. --- :name normalise_comma_joined_encounter_names -UPDATE zone_encounters - SET encounter_name = ( - SELECT mob_name FROM zone_encounter_mobs m - WHERE m.encounter_id = zone_encounters.id - ORDER BY position ASC - LIMIT 1 - ) - WHERE encounter_name LIKE '%,%' - AND EXISTS ( - SELECT 1 FROM zone_encounter_mobs m - WHERE m.encounter_id = zone_encounters.id - ); - --- Insert the parenthesised " (Zone)" form as an alias before stripping it --- from the canonical name, so old references resolve. --- :name normalise_paren_zone_to_alias -INSERT INTO zone_aliases (alias, alias_lower, zone_id) -SELECT name, name_lower, id - FROM zones - WHERE name LIKE '% (Zone)%' -ON CONFLICT DO NOTHING; - --- Strip the " (Zone)" suffix from canonical zone names. --- :name normalise_strip_paren_zone -UPDATE zones - SET name = REPLACE(name, ' (Zone)', ''), - name_lower = REPLACE(name_lower, ' (zone)', '') - WHERE name LIKE '% (Zone)%'; +-- :name meta_upsert +INSERT INTO _meta (key, value) VALUES (%s, %s) +ON CONFLICT (key) DO UPDATE SET value = excluded.value; -- --------------------------------------------------------------------------- -- Column-list fragments @@ -174,9 +35,13 @@ event_name, wiki_url -- Zone CRUD -- --------------------------------------------------------------------------- +-- zones.id is builder-assigned (plain bigint PK, no identity): upsert_zones +-- resolves each source name to its existing id (or allocates max+1) so a +-- metadata rebuild preserves ids — zone_encounters FKs (curator data) stay +-- attached across rebuilds. -- :name upsert_zone INSERT INTO zones ( - name, name_lower, + id, name, name_lower, expansion_short, expansion_name, expansion_year, expansion_confidence, expansion_source, is_persistent_instance, is_endless_persistent, @@ -184,15 +49,16 @@ INSERT INTO zones ( is_live_event, is_city, is_contested, is_deprecated, event_name, wiki_url ) VALUES ( - :name, :name_lower, - :expansion_short, :expansion_name, :expansion_year, - :expansion_confidence, :expansion_source, - :is_persistent_instance, :is_endless_persistent, - :is_tradeskill, :is_pvp, :is_openworld, :is_instance, - :is_live_event, :is_city, :is_contested, :is_deprecated, - :event_name, :wiki_url + %(id)s, %(name)s, %(name_lower)s, + %(expansion_short)s, %(expansion_name)s, %(expansion_year)s, + %(expansion_confidence)s, %(expansion_source)s, + %(is_persistent_instance)s, %(is_endless_persistent)s, + %(is_tradeskill)s, %(is_pvp)s, %(is_openworld)s, %(is_instance)s, + %(is_live_event)s, %(is_city)s, %(is_contested)s, %(is_deprecated)s, + %(event_name)s, %(wiki_url)s ) -ON CONFLICT(name) DO UPDATE SET +ON CONFLICT (id) DO UPDATE SET + name = excluded.name, name_lower = excluded.name_lower, expansion_short = excluded.expansion_short, expansion_name = excluded.expansion_name, @@ -212,45 +78,54 @@ ON CONFLICT(name) DO UPDATE SET event_name = excluded.event_name, wiki_url = excluded.wiki_url; +-- :name select_all_zone_ids +SELECT id, name FROM zones; + +-- Stale-zone prune for the metadata rebuild: zones gone from the cleaned +-- JSON are removed (their types/aliases/encounters cascade). Caller reports +-- the rowcount — a non-zero prune on a routine rebuild deserves a look. +-- :name delete_zones_not_named +DELETE FROM zones WHERE NOT (name = ANY(%s)); + -- :name select_zone_id_by_name -SELECT id FROM zones WHERE name = ?; +SELECT id FROM zones WHERE name = %s; -- :name select_zone_name_and_expansion -SELECT name, expansion_short FROM zones WHERE id = ?; +SELECT name, expansion_short FROM zones WHERE id = %s; -- :name select_zone_name_by_id -SELECT name FROM zones WHERE id = ?; +SELECT name FROM zones WHERE id = %s; -- :name find_zone_by_name_lower -SELECT {cols} FROM zones WHERE name_lower = ? LIMIT 1; +SELECT {cols} FROM zones WHERE name_lower = %s LIMIT 1; -- :name find_zone_by_id -SELECT {cols} FROM zones WHERE id = ?; +SELECT {cols} FROM zones WHERE id = %s; -- :name find_zone_id_by_alias -SELECT zone_id FROM zone_aliases WHERE alias_lower = ? LIMIT 1; +SELECT zone_id FROM zone_aliases WHERE alias_lower = %s LIMIT 1; -- :name find_zone_id_by_alias_aliased -SELECT zone_id AS id FROM zone_aliases WHERE alias_lower = ?; +SELECT zone_id AS id FROM zone_aliases WHERE alias_lower = %s; -- :name select_zone_id_by_name_lower -SELECT id FROM zones WHERE name_lower = ?; +SELECT id FROM zones WHERE name_lower = %s; -- :name list_zones_by_expansion -SELECT {cols} FROM zones WHERE expansion_short = ? ORDER BY name; +SELECT {cols} FROM zones WHERE expansion_short = %s ORDER BY name; -- :name list_zones_by_expansion_typed SELECT {cols} FROM zones -WHERE expansion_short = ? - AND id IN (SELECT zone_id FROM zone_types WHERE type = ?) +WHERE expansion_short = %s + AND id IN (SELECT zone_id FROM zone_types WHERE type = %s) ORDER BY name; -- :name list_zones_by_event -SELECT {cols} FROM zones WHERE is_live_event = 1 AND event_name = ? ORDER BY name; +SELECT {cols} FROM zones WHERE is_live_event = 1 AND event_name = %s ORDER BY name; -- :name list_zones_by_type SELECT {cols} FROM zones -WHERE id IN (SELECT zone_id FROM zone_types WHERE type = ?) +WHERE id IN (SELECT zone_id FROM zone_types WHERE type = %s) ORDER BY name; -- :name list_zones_by_boss @@ -258,170 +133,174 @@ SELECT {cols} FROM zones WHERE id IN ( SELECT e.zone_id FROM zone_encounters e INNER JOIN zone_encounter_mobs m ON m.encounter_id = e.id - WHERE m.mob_name_lower = ? + WHERE m.mob_name_lower = %s ) ORDER BY name; +-- NULLS LAST keeps SQLite's DESC ordering (unknown-year rows sort to the +-- bottom; Postgres DESC would put them first). -- :name list_distinct_expansions SELECT DISTINCT expansion_short, expansion_name, expansion_year FROM zones WHERE expansion_short IS NOT NULL -ORDER BY expansion_year DESC; +ORDER BY expansion_year DESC NULLS LAST; -- :name expansion_counts -SELECT expansion_short, COUNT(*) FROM zones GROUP BY expansion_short ORDER BY 2 DESC; +SELECT expansion_short, COUNT(*) AS n FROM zones GROUP BY expansion_short ORDER BY 2 DESC; -- :name check_zone_exists_for_expansion -SELECT 1 FROM zones WHERE expansion_short = ? LIMIT 1; +SELECT 1 FROM zones WHERE expansion_short = %s LIMIT 1; -- :name select_expansion_name_year SELECT expansion_name AS name, expansion_year AS year FROM zones -WHERE expansion_short = ? LIMIT 1; +WHERE expansion_short = %s LIMIT 1; -- --------------------------------------------------------------------------- -- zone_types CRUD -- --------------------------------------------------------------------------- -- :name delete_zone_types_for_zone -DELETE FROM zone_types WHERE zone_id = ?; +DELETE FROM zone_types WHERE zone_id = %s; -- :name insert_zone_type -INSERT INTO zone_types (zone_id, type) VALUES (?, ?); +INSERT INTO zone_types (zone_id, type) VALUES (%s, %s); -- :name insert_zone_type_or_ignore -INSERT INTO zone_types (zone_id, type) VALUES (?, ?) +INSERT INTO zone_types (zone_id, type) VALUES (%s, %s) ON CONFLICT DO NOTHING; -- :name delete_zone_type -DELETE FROM zone_types WHERE zone_id = ? AND type = ?; +DELETE FROM zone_types WHERE zone_id = %s AND type = %s; -- :name list_types_for_zone -SELECT type FROM zone_types WHERE zone_id = ? ORDER BY type; +SELECT type FROM zone_types WHERE zone_id = %s ORDER BY type; -- :name check_zone_is_raid -SELECT 1 FROM zone_types WHERE zone_id = ? AND type IN ('raid_x4', 'raid_x2') LIMIT 1; +SELECT 1 FROM zone_types WHERE zone_id = %s AND type IN ('raid_x4', 'raid_x2') LIMIT 1; -- --------------------------------------------------------------------------- -- zone_aliases CRUD -- --------------------------------------------------------------------------- -- :name delete_zone_aliases_for_zone -DELETE FROM zone_aliases WHERE zone_id = ?; +DELETE FROM zone_aliases WHERE zone_id = %s; -- :name insert_zone_alias -INSERT INTO zone_aliases (alias, alias_lower, zone_id) VALUES (?, ?, ?); +INSERT INTO zone_aliases (alias, alias_lower, zone_id) VALUES (%s, %s, %s); -- :name list_aliases_for_zone -SELECT alias FROM zone_aliases WHERE zone_id = ? ORDER BY alias; +SELECT alias FROM zone_aliases WHERE zone_id = %s ORDER BY alias; -- --------------------------------------------------------------------------- -- zone_encounters CRUD -- --------------------------------------------------------------------------- -- :name delete_encounters_for_zone -DELETE FROM zone_encounters WHERE zone_id = ?; +DELETE FROM zone_encounters WHERE zone_id = %s; +-- zone_encounters.id is GENERATED ALWAYS AS IDENTITY — runtime inserts never +-- supply it; new ids come back via RETURNING. -- :name insert_encounter -INSERT INTO zone_encounters (zone_id, encounter_name, position, stage, wiki_url) VALUES (?, ?, ?, ?, ?) +INSERT INTO zone_encounters (zone_id, encounter_name, position, stage, wiki_url) VALUES (%s, %s, %s, %s, %s) RETURNING id; -- :name list_encounters_for_zone SELECT id, zone_id, encounter_name, position, stage, wiki_url -FROM zone_encounters WHERE zone_id = ? ORDER BY position; +FROM zone_encounters WHERE zone_id = %s ORDER BY position; -- :name select_encounter_by_id -SELECT id, zone_id, encounter_name, position, stage, wiki_url FROM zone_encounters WHERE id = ?; +SELECT id, zone_id, encounter_name, position, stage, wiki_url FROM zone_encounters WHERE id = %s; -- :name select_encounter_zone_and_name -SELECT id, zone_id, encounter_name FROM zone_encounters WHERE id = ?; +SELECT id, zone_id, encounter_name FROM zone_encounters WHERE id = %s; -- :name select_encounter_zone_id -SELECT zone_id FROM zone_encounters WHERE id = ?; +SELECT zone_id FROM zone_encounters WHERE id = %s; -- :name update_encounter_meta -UPDATE zone_encounters SET encounter_name = ?, stage = ?, wiki_url = ? WHERE id = ?; +UPDATE zone_encounters SET encounter_name = %s, stage = %s, wiki_url = %s WHERE id = %s; -- :name update_encounter_name -UPDATE zone_encounters SET encounter_name = ? WHERE id = ?; +UPDATE zone_encounters SET encounter_name = %s WHERE id = %s; -- :name update_encounter_position -UPDATE zone_encounters SET position = ? WHERE id = ?; +UPDATE zone_encounters SET position = %s WHERE id = %s; -- :name delete_encounter_by_id -DELETE FROM zone_encounters WHERE id = ?; +DELETE FROM zone_encounters WHERE id = %s; -- :name list_zone_encounter_positions -SELECT id, encounter_name, position FROM zone_encounters WHERE zone_id = ?; +SELECT id, encounter_name, position FROM zone_encounters WHERE zone_id = %s; -- :name max_encounter_position_for_zone -SELECT COALESCE(MAX(position), 0) + 1 AS p FROM zone_encounters WHERE zone_id = ?; +SELECT COALESCE(MAX(position), 0) + 1 AS p FROM zone_encounters WHERE zone_id = %s; -- --------------------------------------------------------------------------- -- zone_encounter_mobs CRUD -- --------------------------------------------------------------------------- -- :name insert_encounter_mob -INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (?, ?, ?, ?) +INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (%s, %s, %s, %s) RETURNING id; -- :name insert_encounter_mob_primary -INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (?, ?, ?, 0) +INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) VALUES (%s, %s, %s, 0) RETURNING id; -- :name update_encounter_mob_primary_rename -UPDATE zone_encounter_mobs SET mob_name = ?, mob_name_lower = ? -WHERE encounter_id = ? AND position = 0; +UPDATE zone_encounter_mobs SET mob_name = %s, mob_name_lower = %s +WHERE encounter_id = %s AND position = 0; -- :name list_mobs_for_encounter -SELECT id, mob_name, position FROM zone_encounter_mobs WHERE encounter_id = ? ORDER BY position; +SELECT id, mob_name, position FROM zone_encounter_mobs WHERE encounter_id = %s ORDER BY position; -- :name list_mobs_for_encounter_asc -SELECT id, mob_name, position FROM zone_encounter_mobs WHERE encounter_id = ? ORDER BY position ASC; +SELECT id, mob_name, position FROM zone_encounter_mobs WHERE encounter_id = %s ORDER BY position ASC; -- :name select_primary_mob_name -SELECT mob_name FROM zone_encounter_mobs WHERE encounter_id = ? AND position = 0; +SELECT mob_name FROM zone_encounter_mobs WHERE encounter_id = %s AND position = 0; -- :name select_primary_mob_id_and_name -SELECT id, mob_name FROM zone_encounter_mobs WHERE encounter_id = ? AND position = 0; +SELECT id, mob_name FROM zone_encounter_mobs WHERE encounter_id = %s AND position = 0; -- :name select_mob_by_id -SELECT id, mob_name, position FROM zone_encounter_mobs WHERE id = ?; +SELECT id, mob_name, position FROM zone_encounter_mobs WHERE id = %s; -- :name select_mob_for_update -SELECT encounter_id, mob_name, position FROM zone_encounter_mobs WHERE id = ?; +SELECT encounter_id, mob_name, position FROM zone_encounter_mobs WHERE id = %s; -- :name select_mob_for_promote -SELECT id, encounter_id, mob_name, position FROM zone_encounter_mobs WHERE id = ?; +SELECT id, encounter_id, mob_name, position FROM zone_encounter_mobs WHERE id = %s; -- :name select_mob_encounter_position -SELECT id, encounter_id, position FROM zone_encounter_mobs WHERE id = ?; +SELECT id, encounter_id, position FROM zone_encounter_mobs WHERE id = %s; -- :name update_mob_name -UPDATE zone_encounter_mobs SET mob_name = ?, mob_name_lower = ? WHERE id = ?; +UPDATE zone_encounter_mobs SET mob_name = %s, mob_name_lower = %s WHERE id = %s; -- :name update_mob_position -UPDATE zone_encounter_mobs SET position = ? WHERE id = ?; +UPDATE zone_encounter_mobs SET position = %s WHERE id = %s; -- :name update_mob_position_to_zero -UPDATE zone_encounter_mobs SET position = 0 WHERE id = ?; +UPDATE zone_encounter_mobs SET position = 0 WHERE id = %s; -- :name update_mob_position_to_neg_one -UPDATE zone_encounter_mobs SET position = -1 WHERE id = ?; +UPDATE zone_encounter_mobs SET position = -1 WHERE id = %s; -- :name shift_mobs_negative -UPDATE zone_encounter_mobs SET position = -position - 1 WHERE encounter_id = ?; +UPDATE zone_encounter_mobs SET position = -position - 1 WHERE encounter_id = %s; -- :name shift_mobs_back_positive -UPDATE zone_encounter_mobs SET position = -position WHERE encounter_id = ?; +UPDATE zone_encounter_mobs SET position = -position WHERE encounter_id = %s; -- :name max_mob_position_for_encounter -SELECT COALESCE(MAX(position), -1) + 1 FROM zone_encounter_mobs WHERE encounter_id = ?; +SELECT COALESCE(MAX(position), -1) + 1 AS p FROM zone_encounter_mobs WHERE encounter_id = %s; -- :name count_mobs_for_encounter -SELECT COUNT(*) FROM zone_encounter_mobs WHERE encounter_id = ?; +SELECT COUNT(*) AS n FROM zone_encounter_mobs WHERE encounter_id = %s; -- :name delete_mob_by_id -DELETE FROM zone_encounter_mobs WHERE id = ?; +DELETE FROM zone_encounter_mobs WHERE id = %s; -- --------------------------------------------------------------------------- -- featured_raid_* CRUD @@ -442,7 +321,7 @@ SELECT DISTINCT z.expansion_short AS short, FROM zones z JOIN all_shorts s ON s.short = z.expansion_short WHERE z.expansion_short IS NOT NULL -ORDER BY z.expansion_year DESC, z.expansion_short; +ORDER BY z.expansion_year DESC NULLS LAST, z.expansion_short; -- :name list_available_raid_expansions SELECT DISTINCT z.expansion_short AS short, @@ -457,87 +336,89 @@ WHERE z.expansion_short IS NOT NULL JOIN zones z2 ON z2.id = f.zone_id WHERE z2.expansion_short IS NOT NULL ) -ORDER BY z.expansion_year DESC, z.expansion_short; +ORDER BY z.expansion_year DESC NULLS LAST, z.expansion_short; -- :name insert_featured_raid_expansion -INSERT INTO featured_raid_expansions (expansion_short) VALUES (?) +INSERT INTO featured_raid_expansions (expansion_short) VALUES (%s) ON CONFLICT DO NOTHING; -- :name remove_featured_raid_zones_in_expansion DELETE FROM featured_raid_zones WHERE zone_id IN ( - SELECT id FROM zones WHERE expansion_short = ? + SELECT id FROM zones WHERE expansion_short = %s ); -- :name delete_featured_raid_expansion -DELETE FROM featured_raid_expansions WHERE expansion_short = ?; +DELETE FROM featured_raid_expansions WHERE expansion_short = %s; +-- NULLS FIRST keeps the implicit Uncategorised lane (category IS NULL) at the +-- top — SQLite sorted NULL first by default; Postgres ASC default is last. -- :name list_featured_raid_zones SELECT {cols}, f.position AS featured_position, f.category AS featured_category FROM zones z JOIN featured_raid_zones f ON f.zone_id = z.id -WHERE z.expansion_short = ? -ORDER BY f.category, f.position; +WHERE z.expansion_short = %s +ORDER BY f.category NULLS FIRST, f.position; -- :name list_available_raid_zones SELECT DISTINCT {cols} FROM zones z JOIN zone_types t ON t.zone_id = z.id -WHERE z.expansion_short = ? +WHERE z.expansion_short = %s AND t.type IN ('raid_x4', 'raid_x2') AND z.id NOT IN (SELECT zone_id FROM featured_raid_zones) -ORDER BY z.name; +ORDER BY name; -- :name max_featured_position_uncategorised -SELECT COALESCE(MAX(f.position), -1) +SELECT COALESCE(MAX(f.position), -1) AS p FROM featured_raid_zones f JOIN zones z2 ON z2.id = f.zone_id -WHERE z2.expansion_short = ? AND f.category IS NULL; +WHERE z2.expansion_short = %s AND f.category IS NULL; -- :name insert_featured_raid_zone_uncategorised -INSERT INTO featured_raid_zones (zone_id, position, category) VALUES (?, ?, NULL) +INSERT INTO featured_raid_zones (zone_id, position, category) VALUES (%s, %s, NULL) ON CONFLICT DO NOTHING; -- :name delete_featured_raid_zone_by_name DELETE FROM featured_raid_zones - WHERE zone_id = (SELECT id FROM zones WHERE name_lower = ? LIMIT 1); + WHERE zone_id = (SELECT id FROM zones WHERE name_lower = %s LIMIT 1); -- :name find_featured_zone_id_in_expansion SELECT z.id FROM zones z JOIN featured_raid_zones f ON f.zone_id = z.id -WHERE z.name_lower = ? AND z.expansion_short = ?; +WHERE z.name_lower = %s AND z.expansion_short = %s; -- :name max_category_position -SELECT COALESCE(MAX(position), -1) FROM featured_raid_categories WHERE expansion_short = ?; +SELECT COALESCE(MAX(position), -1) AS p FROM featured_raid_categories WHERE expansion_short = %s; -- :name insert_featured_raid_category_or_ignore -INSERT INTO featured_raid_categories (expansion_short, name, position) VALUES (?, ?, ?) +INSERT INTO featured_raid_categories (expansion_short, name, position) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING; -- :name insert_featured_raid_category -INSERT INTO featured_raid_categories (expansion_short, name, position) VALUES (?, ?, ?); +INSERT INTO featured_raid_categories (expansion_short, name, position) VALUES (%s, %s, %s); -- :name check_featured_raid_category_exists -SELECT 1 FROM featured_raid_categories WHERE expansion_short = ? AND name = ?; +SELECT 1 FROM featured_raid_categories WHERE expansion_short = %s AND name = %s; -- :name update_featured_raid_zone_position_and_category -UPDATE featured_raid_zones SET position = ?, category = ? WHERE zone_id = ?; +UPDATE featured_raid_zones SET position = %s, category = %s WHERE zone_id = %s; -- :name update_featured_raid_zone_position -UPDATE featured_raid_zones SET position = ? WHERE zone_id = ?; +UPDATE featured_raid_zones SET position = %s WHERE zone_id = %s; -- :name update_featured_raid_category_position -UPDATE featured_raid_categories SET position = ? WHERE expansion_short = ? AND name = ?; +UPDATE featured_raid_categories SET position = %s WHERE expansion_short = %s AND name = %s; -- :name list_featured_raid_categories -SELECT name, position FROM featured_raid_categories WHERE expansion_short = ? ORDER BY position; +SELECT name, position FROM featured_raid_categories WHERE expansion_short = %s ORDER BY position; -- :name move_featured_zones_to_null_category UPDATE featured_raid_zones SET category = NULL -WHERE category = ? - AND zone_id IN (SELECT id FROM zones WHERE expansion_short = ?); +WHERE category = %s + AND zone_id IN (SELECT id FROM zones WHERE expansion_short = %s); -- :name delete_featured_raid_category -DELETE FROM featured_raid_categories WHERE expansion_short = ? AND name = ?; +DELETE FROM featured_raid_categories WHERE expansion_short = %s AND name = %s; diff --git a/backend/server/api/act/_shared.py b/backend/server/api/act/_shared.py index 63e313b8..f3bc3f47 100644 --- a/backend/server/api/act/_shared.py +++ b/backend/server/api/act/_shared.py @@ -4,8 +4,6 @@ from __future__ import annotations -import sqlite3 - from fastapi import HTTPException from pydantic import BaseModel @@ -82,41 +80,23 @@ class SpellTimerEntry(BaseModel): # Sync helpers — encounter resolution + DB shapes # --------------------------------------------------------------------------- -_RAIDS_DB_INIT_DONE = False - - -def _ensure_raids_db_inited() -> None: - """Call raids_db.init_db() at most once per process. - - raids_db.init_db() is idempotent (CREATE TABLE IF NOT EXISTS), but - calling it on every trigger read is wasteful. The module-level flag - short-circuits after the first invocation. - """ - global _RAIDS_DB_INIT_DONE - if not _RAIDS_DB_INIT_DONE: - raids_db.init_db().close() - _RAIDS_DB_INIT_DONE = True - - # The synthetic zone that holds contributor-defined trigger CATEGORIES # ("Death Saves", …) — encounters under it are categories, not bosses. -# It exists only in raids.db (verified absent from zones.db), so it rides -# every existing per-encounter route, the XML import/export, and the app -# pack with zero special-casing beyond the resolution branch below. +# It exists only in the raids schema (verified absent from zones.db), so it +# rides every existing per-encounter route, the XML import/export, and the +# app pack with zero special-casing beyond the resolution branch below. GENERAL_ZONE = "General" def _resolve_general_sync(position: int) -> tuple[str, str, int] | None: - """Resolve a General-zone category purely against raids.db — there is - no zones_db backing, and no lazy-create (categories are created - explicitly via the categories API).""" - _ensure_raids_db_inited() - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + """Resolve a General-zone category purely against the raids schema — + there is no zones_db backing, and no lazy-create (categories are + created explicitly via the categories API).""" + with raids_db.init_db() as conn: row = conn.execute( """SELECT e.id, e.mob_name FROM raid_encounters e JOIN raid_zones z ON z.id = e.raid_zone_id - WHERE z.zone_name_lower = ? AND e.position = ?""", + WHERE z.zone_name_lower = %s AND e.position = %s""", (GENERAL_ZONE.lower(), position), ).fetchone() if row is None: @@ -149,20 +129,10 @@ def _resolve_encounter_sync(zone_name: str, position: int) -> tuple[str, str, in if mob_name is None: return None - # Find or lazy-create the raids_db rows. - # - # _ensure_raids_db_inited() is idempotent (CREATE TABLE IF NOT EXISTS) - # and is the only thing that ensures the act_triggers / act_spell_timers - # tables exist on an older raids.db that was seeded before they were - # added to the schema. Without this, a viewer hitting the GET endpoint - # against a stale DB sees "no such table: act_triggers" → 500. - # After the first call per process it is a no-op (module-level flag). - _ensure_raids_db_inited() - - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + # Find or lazy-create the raids rows. + with raids_db.init_db() as conn: zrow = conn.execute( - "SELECT id FROM raid_zones WHERE zone_name_lower = ?", + "SELECT id FROM raid_zones WHERE zone_name_lower = %s", (canonical_zone.lower(),), ).fetchone() zone_id: int @@ -179,7 +149,7 @@ def _resolve_encounter_sync(zone_name: str, position: int) -> tuple[str, str, in zone_id = zrow["id"] erow = conn.execute( - "SELECT id FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?", + "SELECT id FROM raid_encounters WHERE raid_zone_id = %s AND mob_name_lower = %s", (zone_id, mob_name.lower()), ).fetchone() if erow is None: diff --git a/backend/server/api/act/categories.py b/backend/server/api/act/categories.py index db9f9de8..ac641548 100644 --- a/backend/server/api/act/categories.py +++ b/backend/server/api/act/categories.py @@ -13,16 +13,12 @@ from __future__ import annotations import logging -import sqlite3 from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field from backend.eq2db.raids import catalogue as raids_db -from backend.server.api.act._shared import ( - GENERAL_ZONE, - _ensure_raids_db_inited, -) +from backend.server.api.act._shared import GENERAL_ZONE from backend.server.auth_deps import require_editor from backend.server.core.audit_log import audit_log from backend.server.core.executor import run_sync @@ -45,30 +41,25 @@ class CategoryUpsertRequest(BaseModel): def _list_categories_sync() -> list[dict]: - _ensure_raids_db_inited() - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row - rows = conn.execute( + with raids_db.init_db() as conn: + return conn.execute( """SELECT e.mob_name AS name, e.position, (SELECT COUNT(*) FROM act_triggers t WHERE t.raid_encounter_id = e.id) AS trigger_count, (SELECT COUNT(*) FROM act_spell_timers s WHERE s.raid_encounter_id = e.id) AS spell_timer_count FROM raid_encounters e JOIN raid_zones z ON z.id = e.raid_zone_id - WHERE z.zone_name_lower = ? + WHERE z.zone_name_lower = %s ORDER BY e.position, e.mob_name""", (GENERAL_ZONE.lower(),), ).fetchall() - return [dict(r) for r in rows] def _create_category_sync(name: str) -> tuple[int, str] | None: """Create the category (and the General zone row on first use). Returns (position, name) or None when the name already exists.""" - _ensure_raids_db_inited() - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: zrow = conn.execute( - "SELECT id FROM raid_zones WHERE zone_name_lower = ?", + "SELECT id FROM raid_zones WHERE zone_name_lower = %s", (GENERAL_ZONE.lower(),), ).fetchone() zone_id = ( @@ -82,15 +73,17 @@ def _create_category_sync(name: str) -> tuple[int, str] | None: ) ) exists = conn.execute( - "SELECT 1 FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?", + "SELECT 1 FROM raid_encounters WHERE raid_zone_id = %s AND mob_name_lower = %s", (zone_id, name.lower()), ).fetchone() if exists is not None: return None - position = conn.execute( - "SELECT COALESCE(MAX(position) + 1, 0) FROM raid_encounters WHERE raid_zone_id = ?", - (zone_id,), - ).fetchone()[0] + position = raids_db.fetchval( + conn.execute( + "SELECT COALESCE(MAX(position) + 1, 0) AS next_pos FROM raid_encounters WHERE raid_zone_id = %s", + (zone_id,), + ) + ) raids_db.upsert_raid_encounter( conn, raid_zone_id=zone_id, @@ -105,13 +98,11 @@ def _create_category_sync(name: str) -> tuple[int, str] | None: def _rename_category_sync(position: int, new_name: str) -> bool | None: """Rename by position. True = renamed, False = position unknown, None = the new name collides with an existing category.""" - _ensure_raids_db_inited() - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: row = conn.execute( """SELECT e.mob_name FROM raid_encounters e JOIN raid_zones z ON z.id = e.raid_zone_id - WHERE z.zone_name_lower = ? AND e.position = ?""", + WHERE z.zone_name_lower = %s AND e.position = %s""", (GENERAL_ZONE.lower(), position), ).fetchone() if row is None: @@ -119,7 +110,7 @@ def _rename_category_sync(position: int, new_name: str) -> bool | None: collision = conn.execute( """SELECT 1 FROM raid_encounters e JOIN raid_zones z ON z.id = e.raid_zone_id - WHERE z.zone_name_lower = ? AND e.mob_name_lower = ? AND e.position != ?""", + WHERE z.zone_name_lower = %s AND e.mob_name_lower = %s AND e.position != %s""", (GENERAL_ZONE.lower(), new_name.lower(), position), ).fetchone() if collision is not None: @@ -133,14 +124,11 @@ def _rename_category_sync(position: int, new_name: str) -> bool | None: def _delete_category_sync(position: int) -> bool: - _ensure_raids_db_inited() - with sqlite3.connect(raids_db.path) as conn: - conn.execute("PRAGMA foreign_keys = ON") - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: row = conn.execute( """SELECT e.mob_name FROM raid_encounters e JOIN raid_zones z ON z.id = e.raid_zone_id - WHERE z.zone_name_lower = ? AND e.position = ?""", + WHERE z.zone_name_lower = %s AND e.position = %s""", (GENERAL_ZONE.lower(), position), ).fetchone() if row is None: diff --git a/backend/server/api/act/pack.py b/backend/server/api/act/pack.py index a475b1e6..930625b9 100644 --- a/backend/server/api/act/pack.py +++ b/backend/server/api/act/pack.py @@ -15,7 +15,6 @@ from __future__ import annotations -import sqlite3 import time from fastapi import APIRouter @@ -26,7 +25,6 @@ from backend.server.api.act._shared import ( SpellTimerEntry, TriggerEntry, - _ensure_raids_db_inited, _spell_row_to_entry, _trigger_row_to_entry, ) @@ -57,9 +55,7 @@ class ActPack(BaseModel): def _build_pack_sync() -> ActPack: - _ensure_raids_db_inited() - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: encounters = conn.execute( """ SELECT e.id AS encounter_id, e.mob_name, e.position, diff --git a/backend/server/api/act/spell_timers.py b/backend/server/api/act/spell_timers.py index ac7fb82b..610e54bb 100644 --- a/backend/server/api/act/spell_timers.py +++ b/backend/server/api/act/spell_timers.py @@ -10,8 +10,7 @@ from __future__ import annotations -import sqlite3 - +import psycopg.errors from fastapi import APIRouter, Depends, HTTPException, Response from pydantic import BaseModel, Field, field_validator @@ -158,7 +157,7 @@ def _write() -> int: try: new_id = await run_sync(_write) - except sqlite3.IntegrityError as exc: + except psycopg.errors.UniqueViolation as exc: raise HTTPException( status_code=409, detail=f"A spell timer named {body.name!r} already exists for this encounter", @@ -222,7 +221,7 @@ def _write() -> int: try: await run_sync(_write) - except sqlite3.IntegrityError as exc: + except psycopg.errors.UniqueViolation as exc: raise HTTPException( status_code=409, detail=f"Renaming to {body.name!r} would clash with another timer for this encounter", diff --git a/backend/server/api/act/triggers.py b/backend/server/api/act/triggers.py index 35623d02..3da2367b 100644 --- a/backend/server/api/act/triggers.py +++ b/backend/server/api/act/triggers.py @@ -13,8 +13,6 @@ from __future__ import annotations -import sqlite3 - from fastapi import APIRouter, Depends, HTTPException, Response from pydantic import BaseModel, Field @@ -89,27 +87,23 @@ class ImportXmlResponse(BaseModel): def _trigger_already_exists_sync(encounter_id: int, regex: str, sound_data: str) -> int | None: """Idempotency: same (encounter, regex, sound) is treated as identity. Returns the existing row id, or None.""" - if not raids_db.path.exists(): - return None - with sqlite3.connect(raids_db.path) as conn: + with raids_db.init_db() as conn: row = conn.execute( - "SELECT id FROM act_triggers WHERE raid_encounter_id = ? AND regex = ? AND sound_data = ?", + "SELECT id FROM act_triggers WHERE raid_encounter_id = %s AND regex = %s AND sound_data = %s", (encounter_id, regex, sound_data), ).fetchone() - return int(row[0]) if row else None + return int(row["id"]) if row else None def _spell_timer_id_for_name_sync(encounter_id: int, name: str) -> int | None: """Find an existing spell timer's id by lowercase name within the encounter — UNIQUE (encounter_id, name_lower) makes this safe.""" - if not raids_db.path.exists(): - return None - with sqlite3.connect(raids_db.path) as conn: + with raids_db.init_db() as conn: row = conn.execute( - "SELECT id FROM act_spell_timers WHERE raid_encounter_id = ? AND name_lower = ?", + "SELECT id FROM act_spell_timers WHERE raid_encounter_id = %s AND name_lower = %s", (encounter_id, name.lower()), ).fetchone() - return int(row[0]) if row else None + return int(row["id"]) if row else None # --------------------------------------------------------------------------- diff --git a/backend/server/api/raid_strategies.py b/backend/server/api/raid_strategies.py index ef9b654f..5c40d54a 100644 --- a/backend/server/api/raid_strategies.py +++ b/backend/server/api/raid_strategies.py @@ -7,8 +7,8 @@ GET /api/zones/{zone}/overview/revisions (history) Read-write surface for raid strategy markdown — per-encounter strategies and -zone-level overview. Bodies live in ``census/raids_db.py`` (separate SQLite -file from the zones DB). Write gate is ``require_editor`` from +zone-level overview. Bodies live in ``backend/eq2db/raids.py`` (the raids +Postgres schema). Write gate is ``require_editor`` from ``web/auth_deps.py`` (admin / contributor — see that module's docstring for the role model). @@ -33,8 +33,8 @@ from __future__ import annotations import logging -import sqlite3 import time +from typing import Any from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field @@ -176,10 +176,7 @@ def _read_revisions_sync(zone_name: str, encounter_name: str) -> list[dict]: Returns ``[]`` if the encounter doesn't exist in raids_db yet (never had a strategy written) — the route surfaces that as a 200 with an empty list, matching the "show history" disclosure's no-op state.""" - if not raids_db.path.exists(): - return [] - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: zrow = conn.execute(_SQL["select_raid_zone_id_by_name"], (zone_name.lower(),)).fetchone() if zrow is None: return [] @@ -200,10 +197,7 @@ def _read_overview_sync(zone_name: str) -> dict | None: None when no row exists yet OR when overview_md is empty — same semantics as the encounter helpers, lets the route 404 cleanly and the UI fall back to the empty state.""" - if not raids_db.path.exists(): - return None - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: row = conn.execute( _SQL["select_raid_zone_overview"], (zone_name.lower(),), @@ -214,7 +208,7 @@ def _read_overview_sync(zone_name: str) -> dict | None: def _create_overview_sync( - conn: sqlite3.Connection, + conn: Any, *, zone_name: str, markdown: str, @@ -243,12 +237,12 @@ def _create_overview_sync( ).fetchone() conn.execute( _SQL["insert_raid_zone_revision_first"], - (zone_id_row[0], now, editor_discord_id, markdown, edit_note), + (zone_id_row["id"], now, editor_discord_id, markdown, edit_note), ) def _update_overview_sync( - conn: sqlite3.Connection, + conn: Any, *, zone_id: int, zone_name: str, @@ -312,16 +306,15 @@ def _write_overview_sync( else: _update_overview_sync( conn, - zone_id=int(existing[0]), + zone_id=int(existing["id"]), zone_name=zone_name, - prev_md=existing[1], + prev_md=existing["overview_md"], markdown=markdown, editor_discord_id=editor_discord_id, edit_note=edit_note, now=now, ) conn.commit() - conn.row_factory = sqlite3.Row row = conn.execute( _SQL["select_raid_zone_overview"], (zone_name.lower(),), @@ -333,11 +326,8 @@ def _write_overview_sync( def _read_strategy_sync(zone_name: str, encounter_name: str) -> dict | None: """Look up an existing strategy row. Returns None if none exists yet.""" - if not raids_db.path.exists(): - return None - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row - # Find the raid_zones id (loose name match — the strategy DB stores + with raids_db.init_db() as conn: + # Find the raid_zones id (loose name match — the strategy store keeps # the canonical zone_name verbatim, so a case-insensitive lower-match # is robust against any alias canonicalisation drift). zrow = conn.execute(_SQL["select_raid_zone_id_by_name"], (zone_name.lower(),)).fetchone() @@ -365,8 +355,7 @@ def _write_strategy_sync( """Upsert a strategy row. Auto-creates the raid_zones parent on first write. Returns the fresh row as a dict (same shape as ``_read_strategy_sync``).""" - # init_db is idempotent — safe to call every time. Also creates the parent - # data/raids/ directory. + # init_db returns a pooled connection scoped to the raids schema. conn = raids_db.init_db() try: zone_id = raids_db.upsert_raid_zone( @@ -387,7 +376,6 @@ def _write_strategy_sync( ) # Re-read so we return the freshly-merged row (last_edited_at, source, # etc. — easier than reconstructing it client-side). - conn.row_factory = sqlite3.Row row = conn.execute( _SQL["select_encounter_strategy"], (zone_id, encounter_name.lower()), @@ -561,10 +549,7 @@ def _read_zone_revisions_sync(zone_name: str) -> list[dict]: """All revision rows for a zone's overview, newest first. Returns [] if the zone has no raid_zones row OR no revisions yet.""" - if not raids_db.path.exists(): - return [] - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: zrow = conn.execute( _SQL["select_raid_zone_id_by_name"], (zone_name.lower(),), diff --git a/backend/server/api/raid_strategies.sql b/backend/server/api/raid_strategies.sql index e51b7f28..66c142fa 100644 --- a/backend/server/api/raid_strategies.sql +++ b/backend/server/api/raid_strategies.sql @@ -1,5 +1,5 @@ -- SQL for backend/server/api/raid_strategies.py — encounter / zone --- overview + strategy read/write + revisions. +-- overview + strategy read/write + revisions (psycopg, raids schema). -- -- All queries are fixed shapes (no dynamic WHERE assembly). Several -- queries appear in multiple Python helpers because each runs in its @@ -11,14 +11,14 @@ -- --------------------------------------------------------------------------- -- :name select_raid_zone_id_by_name -SELECT id FROM raid_zones WHERE zone_name_lower = ?; +SELECT id FROM raid_zones WHERE zone_name_lower = %s; -- :name select_raid_zone_id_and_overview -SELECT id, overview_md FROM raid_zones WHERE zone_name_lower = ?; +SELECT id, overview_md FROM raid_zones WHERE zone_name_lower = %s; -- :name select_raid_zone_overview SELECT zone_name, overview_md, source, last_edited_at, last_edited_by -FROM raid_zones WHERE zone_name_lower = ?; +FROM raid_zones WHERE zone_name_lower = %s; -- --------------------------------------------------------------------------- -- raid_zones overview writes @@ -27,17 +27,17 @@ FROM raid_zones WHERE zone_name_lower = ?; -- :name update_raid_zone_audit_fields -- Stamps last_edited_at + last_edited_by after the upsert_raid_zone helper -- runs (which doesn't touch the audit columns itself). -UPDATE raid_zones SET last_edited_at = ?, last_edited_by = ? -WHERE zone_name_lower = ?; +UPDATE raid_zones SET last_edited_at = %s, last_edited_by = %s +WHERE zone_name_lower = %s; -- :name update_raid_zone_overview -- Existing-row overview replace + audit-field bump in one statement. UPDATE raid_zones SET - overview_md = ?, - source = ?, - last_edited_at = ?, - last_edited_by = ? -WHERE id = ?; + overview_md = %s, + source = %s, + last_edited_at = %s, + last_edited_by = %s +WHERE id = %s; -- --------------------------------------------------------------------------- -- raid_zone_revisions @@ -47,20 +47,20 @@ WHERE id = ?; -- First-ever revision for a zone overview — before_md is intentionally NULL. INSERT INTO raid_zone_revisions (raid_zone_id, edited_at, edited_by, before_md, after_md, edit_note) -VALUES (?, ?, ?, NULL, ?, ?); +VALUES (%s, %s, %s, NULL, %s, %s); -- :name insert_raid_zone_revision -- Subsequent revisions carry the previous markdown in before_md. INSERT INTO raid_zone_revisions (raid_zone_id, edited_at, edited_by, before_md, after_md, edit_note) -VALUES (?, ?, ?, ?, ?, ?); +VALUES (%s, %s, %s, %s, %s, %s); -- --------------------------------------------------------------------------- -- raid_encounters -- --------------------------------------------------------------------------- -- :name select_encounter_id_by_zone_mob -SELECT id FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?; +SELECT id FROM raid_encounters WHERE raid_zone_id = %s AND mob_name_lower = %s; -- :name select_encounter_strategy -- Full strategy row for an encounter — used by both the read path and the @@ -68,4 +68,4 @@ SELECT id FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?; SELECT id, mob_name, position, strategy_md, source, last_edited_at, last_edited_by FROM raid_encounters -WHERE raid_zone_id = ? AND mob_name_lower = ?; +WHERE raid_zone_id = %s AND mob_name_lower = %s; diff --git a/backend/server/api/rankings.py b/backend/server/api/rankings.py index 1a4fc12c..87ef1a4e 100644 --- a/backend/server/api/rankings.py +++ b/backend/server/api/rankings.py @@ -14,7 +14,6 @@ import asyncio import hashlib import logging -import sqlite3 import unicodedata from collections import defaultdict from functools import lru_cache @@ -326,14 +325,13 @@ def _raid_boss_names() -> tuple[str, ...]: """Sorted, normalised (``_normalise_boss_key``) mob names of every curated raid-zone encounter — what ``GET /api/zones/raid-bosses`` ships to the desktop parser so it can decide client-side whether a fight is a - raid boss. Empty when zones.db is absent. Cleared by + raid boss. Empty when no rosters are curated yet. Cleared by invalidate_zones_cache() on curator edits.""" - path = zones_db.path - if not path.exists(): - return () - conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + conn = zones_db.init_db() try: - names = {_normalise_boss_key(r[0]) for r in conn.execute(_SQL["list_raid_boss_mob_names"])} + names = { + _normalise_boss_key(r["mob_name_lower"]) for r in conn.execute(_SQL["list_raid_boss_mob_names"]).fetchall() + } finally: conn.close() return tuple(sorted(n for n in names if n)) @@ -350,7 +348,7 @@ def raid_boss_pack() -> dict: @lru_cache(maxsize=1) def _cached_zones_data() -> tuple[dict[str, list[tuple[str, str]]], list[dict], list[dict], set[str]]: - """Authoritative zone/boss data from zones.db, built once per process. + """Authoritative zone/boss data from the zones schema, built once per process. Returns (boss_index, raid_tree, dungeon_tree, curated_zone_names): * boss_index: ``mob_name_lower -> [(canonical_zone, encounter_name), ...]`` @@ -366,36 +364,42 @@ def _cached_zones_data() -> tuple[dict[str, list[tuple[str, str]]], list[dict], that happens to have bosses (which they all do, post-PR #36) would show under the "Raids" dropdown alongside the actual raids. - Empty when zones.db is absent (dev/pre-upload), so everything falls back - to the is_boss heuristic and parse-derived dropdowns. + Empty when no rosters are curated yet (dev/pre-seed), so everything falls + back to the is_boss heuristic and parse-derived dropdowns. PROCESS-LOCAL: this LRU lives in one Python process. invalidate_zones_cache() only clears it on the worker that handled the mutation; sibling workers serve stale data until they happen to evict. A startup assertion in web/app.py:_startup pins WEB_CONCURRENCY=1 so this is safe — if that - assertion is ever loosened, swap this for an mtime-based reload (compare - ``zones.db.stat().st_mtime`` against the cached value on each call) or - move invalidation to a Redis-backed fan-out.""" - path = zones_db.path - if not path.exists(): - return {}, [], [], set() - conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + assertion is ever loosened, move invalidation to a Redis-backed fan-out + (or re-read on a short TTL).""" + conn = zones_db.init_db() try: boss_index: dict[str, list[tuple[str, str]]] = defaultdict(list) - for mob_lower, zname, ename in conn.execute(_SQL["list_all_zone_encounter_mobs"]): - boss_index[_normalise_boss_key(mob_lower)].append((zname, ename)) + for r in conn.execute(_SQL["list_all_zone_encounter_mobs"]).fetchall(): + boss_index[_normalise_boss_key(r["mob_name_lower"])].append((r["name"], r["encounter_name"])) def _tree_for_type(type_token: str) -> list[dict]: """Materialise the (zone, expansion, bosses) ordered list for one zone-type token. Joins zones → zone_types so the same query powers both the raid and dungeon trees with no duplication.""" out: list[dict] = [] - for zid, zname, exp, exp_name in conn.execute( + for zr in conn.execute( _SQL["list_zones_by_type_with_encounters"], (type_token,), - ): - bosses = [r[0] for r in conn.execute(_SQL["list_encounter_names_for_zone"], (zid,))] - out.append({"zone": zname, "expansion": exp, "expansion_name": exp_name, "bosses": bosses}) + ).fetchall(): + bosses = [ + r["encounter_name"] + for r in conn.execute(_SQL["list_encounter_names_for_zone"], (zr["id"],)).fetchall() + ] + out.append( + { + "zone": zr["name"], + "expansion": zr["expansion_short"], + "expansion_name": zr["expansion_name"], + "bosses": bosses, + } + ) return out raid_tree = _tree_for_type("raid_x4") @@ -413,16 +417,14 @@ def _encounter_required_mobs() -> dict[tuple[str, str], frozenset[str]]: """(canonical zone, encounter name) → normalised mob keys, for curated encounters with MULTIPLE mobs — the anti-cut-parse gate's requirement lists. Single-mob encounters carry no requirement: the fight ends when - the mob dies, so there is nothing to cut. Empty when zones.db is absent - (dev/tests). Cleared by invalidate_zones_cache() on curator edits.""" - path = zones_db.path - if not path.exists(): - return {} + the mob dies, so there is nothing to cut. Empty when no rosters are + curated yet (dev/tests). Cleared by invalidate_zones_cache() on curator + edits.""" by_enc: dict[tuple[str, str], set[str]] = defaultdict(set) - conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + conn = zones_db.init_db() try: - for mob_lower, zname, ename in conn.execute(_SQL["list_all_zone_encounter_mobs"]): - by_enc[(zname, ename)].add(_normalise_boss_key(mob_lower)) + for r in conn.execute(_SQL["list_all_zone_encounter_mobs"]).fetchall(): + by_enc[(r["name"], r["encounter_name"])].add(_normalise_boss_key(r["mob_name_lower"])) finally: conn.close() return {k: frozenset(v) for k, v in by_enc.items() if len(v) > 1} @@ -594,20 +596,22 @@ def _build_filters(kills: list[dict]) -> dict: def _zone_expansion_map() -> dict[str, str]: - """{canonical zone name lower: expansion short} from zones.db. Cached — - cleared by invalidate_zones_cache alongside the boss trees.""" + """{canonical zone name lower: expansion short} from the zones schema. + Cached — cleared by invalidate_zones_cache alongside the boss trees. + + The old SQLite version swallowed sqlite3.Error because a dev checkout + might simply not have the zones.db file; on Postgres the schema is + guaranteed by migrations, so a failure here is a real fault and + propagates.""" global _zone_xpac_cache if _zone_xpac_cache is None: - mapping: dict[str, str] = {} + conn = zones_db.init_db() try: - with sqlite3.connect(zones_db.path) as conn: - for name_lower, short in conn.execute( - "SELECT name_lower, expansion_short FROM zones WHERE expansion_short IS NOT NULL" - ): - mapping[name_lower] = short - except sqlite3.Error: # zones.db missing locally — no lock possible - pass - _zone_xpac_cache = mapping + _zone_xpac_cache = { + r["name_lower"]: r["expansion_short"] for r in conn.execute(_SQL["map_zone_expansions"]).fetchall() + } + finally: + conn.close() return _zone_xpac_cache diff --git a/backend/server/api/rankings.sql b/backend/server/api/rankings.sql index d78dd503..104f3a45 100644 --- a/backend/server/api/rankings.sql +++ b/backend/server/api/rankings.sql @@ -7,7 +7,7 @@ -- can compose the two via load_sql at module-load. -- --------------------------------------------------------------------------- --- zones.db — boss-tree builders for the leaderboard sidebar +-- zones schema — boss-tree builders for the leaderboard sidebar -- --------------------------------------------------------------------------- -- :name list_all_zone_encounter_mobs @@ -31,17 +31,22 @@ WHERE t.type IN ('raid_x4', 'raid_x2'); -- :name list_zones_by_type_with_encounters -- Zones tagged with a given zone-type token that have at least one --- encounter. Ordered newest-expansion-first within type. +-- encounter. Ordered newest-expansion-first within type (NULLS LAST keeps +-- SQLite's DESC ordering for unknown-year rows). SELECT z.id, z.name, z.expansion_short, z.expansion_name FROM zones z JOIN zone_types t ON t.zone_id = z.id -WHERE t.type = ? +WHERE t.type = %s AND z.id IN (SELECT DISTINCT zone_id FROM zone_encounters) -ORDER BY z.expansion_year DESC, z.name; +ORDER BY z.expansion_year DESC NULLS LAST, z.name; -- :name list_encounter_names_for_zone SELECT encounter_name FROM zone_encounters -WHERE zone_id = ? ORDER BY position; +WHERE zone_id = %s ORDER BY position; + +-- :name map_zone_expansions +-- Era-lock map: every canonical zone's expansion short, keyed lowercased. +SELECT name_lower, expansion_short FROM zones WHERE expansion_short IS NOT NULL; -- --------------------------------------------------------------------------- -- parses schema (Postgres) — winning-encounters scan + player_count refresh diff --git a/backend/server/api/zones.py b/backend/server/api/zones.py index eb22a529..929fe7a4 100644 --- a/backend/server/api/zones.py +++ b/backend/server/api/zones.py @@ -9,22 +9,19 @@ ``scripts/dev/eq2_raid_bosses.review.txt`` and rebuilt with ``scripts/build_zones_db.py``. -Sync sqlite calls (``zones_db.list_by_expansion`` / ``find_by_name``) are +Sync DB calls (``zones_db.list_by_expansion`` / ``find_by_name``) are offloaded with ``run_in_executor`` so they don't block the event loop — same pattern as ``recipes.py`` and ``classes.py``. """ from __future__ import annotations -import sqlite3 - from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from pydantic import BaseModel from backend.eq2db.zones import catalogue as zones_db from backend.server.api.rankings import raid_boss_pack from backend.server.auth_deps import require_user_session -from backend.server.constants import SQLITE_VAR_CHUNK_SAFE from backend.server.core.executor import run_sync from backend.server.core.primary_guild import cached_primary_guild from backend.server.core.session_user import SessionUser @@ -285,12 +282,9 @@ def _compute_progress_sync(guild_name: str) -> dict[str, list[KilledEncounter]]: ``zone_encounter_mobs.mob_name_lower`` — solo bosses match directly, group encounters collapse (any one mob ACT logs counts the whole encounter as cleared). Aggregation happens in Python rather than SQL because the parses - data lives in Postgres while the zones data is a local SQLite file — a - Python stitch beats any cross-database join contraption. + and zones data live in separate Postgres schemas on separate connections — + a Python stitch keeps the two reads independent. """ - if not zones_db.path.exists(): - return {} - # Pull every winning row for the guild as (id, title_lower, started_at). # We need the timestamp + id to surface "last kill" — a DISTINCT title pass # wouldn't be enough. @@ -307,23 +301,17 @@ def _compute_progress_sync(guild_name: str) -> dict[str, list[KilledEncounter]]: if not kills: return {} - # Build a mob_lower → (zone, encounter) map for every mob we actually need. - # SQLite's variable limit is 999 by default; chunk to stay well clear. + # Build a mob_lower → (zone, encounter) map for every mob we actually + # need — one = ANY(array) query (no SQLite variable-limit chunking). title_set = {t for _, t, _ in kills} mob_to_enc: dict[str, tuple[str, str]] = {} - with sqlite3.connect(zones_db.path) as zconn: - zconn.execute("PRAGMA query_only = ON") - zconn.row_factory = sqlite3.Row - titles_list = list(title_set) - for i in range(0, len(titles_list), SQLITE_VAR_CHUNK_SAFE): - chunk = titles_list[i : i + SQLITE_VAR_CHUNK_SAFE] - placeholders = ",".join("?" * len(chunk)) - rows = zconn.execute( - _SQL["match_encounter_mobs_by_titles_chunk"].format(placeholders=placeholders), - chunk, - ).fetchall() - for r in rows: - mob_to_enc[r["mob_lower"]] = (r["zone_name"], r["encounter_name"]) + zconn = zones_db.init_db() + try: + rows = zconn.execute(_SQL["match_encounter_mobs_by_titles"], (list(title_set),)).fetchall() + finally: + zconn.close() + for r in rows: + mob_to_enc[r["mob_lower"]] = (r["zone_name"], r["encounter_name"]) # Per-encounter aggregate: kill count + most-recent kill (id + timestamp). agg: dict[tuple[str, str], dict] = {} diff --git a/backend/server/api/zones.sql b/backend/server/api/zones.sql index 2d203b2b..5275adfd 100644 --- a/backend/server/api/zones.sql +++ b/backend/server/api/zones.sql @@ -1,10 +1,6 @@ -- SQL for backend/server/api/zones.py — zone routes (single zone, list, --- progress-by-guild). --- --- `match_encounter_mobs_by_titles_chunk` is parameterised over its IN-clause --- because SQLite needs a known placeholder count. The caller builds the --- chunked IN list as ",".join("?" * len(chunk)) and `.format(placeholders=...)`s --- it in — same template trick used in backend/eq2db/zones.sql for `{cols}`. +-- progress-by-guild). Both the parses and zones queries run against +-- Postgres (%s placeholders), each on its own schema-scoped connection. -- :name most_recent_parsed_guild -- Most recent non-null guild_name this user has uploaded a parse for. @@ -19,14 +15,13 @@ ORDER BY started_at DESC LIMIT 1; SELECT id, title, started_at FROM encounters WHERE guild_name = %s AND success_level = 1 AND hidden_at IS NULL; --- :name match_encounter_mobs_by_titles_chunk --- For a chunk of mob_lower titles, resolve each to its (zone, encounter) --- pair. Caller chunks by SQLITE_VAR_CHUNK_SAFE to stay under SQLite's --- variable limit (default 999) and templates the {placeholders} in. +-- :name match_encounter_mobs_by_titles +-- For a set of mob_lower titles (bound as one text[] parameter), resolve +-- each to its (zone, encounter) pair. Zones schema. SELECT m.mob_name_lower AS mob_lower, z.name AS zone_name, e.encounter_name AS encounter_name FROM zone_encounter_mobs m JOIN zone_encounters e ON e.id = m.encounter_id JOIN zones z ON z.id = e.zone_id -WHERE m.mob_name_lower IN ({placeholders}); +WHERE m.mob_name_lower = ANY(%s); diff --git a/backend/server/app.py b/backend/server/app.py index bf7fd376..ddf4deb1 100644 --- a/backend/server/app.py +++ b/backend/server/app.py @@ -499,15 +499,13 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: # upload's request pays that cost on the request thread. from backend.server.parses.db import store as parses_db - parses_db.init_db() - # Initialise zones.db on startup too so admin-curation tables - # (featured_raid_expansions / featured_raid_zones) exist on - # pre-migration DBs without requiring a rebuild via the seed - # script. init_db is CREATE TABLE IF NOT EXISTS throughout, so - # this is safe on populated zones.db files. - from backend.eq2db import zones as zones_db - - zones_db.catalogue.init_db().close() + # Warm one parses checkout + release it (init_db returns a pooled + # proxy now — holding it would leak a sync-pool slot for the + # process lifetime). + parses_db.init_db().close() + # (zones moved to Postgres — its schema is owned by db/migrations/ + # 0004_zones.sql, applied by the pg_migrate.run above; no per-start + # init needed.) # Initialise recipes.db synchronously so the out_level column (and any # future migrations) exist BEFORE the first recipe read. The search and # eq2db find_* paths SELECT out_level directly and do not run init_db diff --git a/backend/server/metrics.py b/backend/server/metrics.py index 1e30ff16..808bc339 100644 --- a/backend/server/metrics.py +++ b/backend/server/metrics.py @@ -15,7 +15,6 @@ import logging import os import re -import sqlite3 import time from prometheus_client import ( @@ -186,66 +185,13 @@ def parse_age_bucket(started_at: int | None, now: int) -> str: class _DBCollector(Collector): """ - Custom collector that runs fast COUNT queries against the local SQLite DBs - each time Prometheus scrapes /metrics. SQLite COUNTs on indexed tables in - the few-thousand-row range are sub-millisecond, so blocking the collector - is fine. - - A 30-second scrape interval × ~12 queries × <1 ms each is ~12 ms/scrape - of total DB work — well under any threshold worth caching for. - - BE-229: connections are kept open between scrapes (ro URI mode) to avoid - the open/close overhead every 30 s. A failed connection is retried on the - next scrape (the dict slot is cleared on exception). + Custom collector that runs fast COUNT queries against the migrated + Postgres family schemas each time Prometheus scrapes /metrics. Tiny + COUNTs per scrape; a short-lived pooled checkout per family keeps the + collector simple. (The P2 metrics split replaces this with 60s-cached + counts.) """ - def __init__(self) -> None: - self._conns: dict[str, sqlite3.Connection] = {} - - def _get_conn(self, name: str, path: object) -> sqlite3.Connection | None: - """Return a cached read-only connection, opening it lazily. - - ``check_same_thread=False`` is required because Prometheus scrapes can - arrive on different threads (uvicorn worker vs the request thread the - first scrape happened on). Safe for our use because: - 1. Connections are opened in read-only (``?mode=ro``) URI mode — no - writes ever happen on these connections, so SQLite's serialised - write mode isn't entered. - 2. SQLite itself supports concurrent reads from multiple threads; - ``check_same_thread`` is Python's conservative default safety - guard, not a SQLite-level constraint. - - Without this flag, scrape #2 from a different thread than scrape #1 - crashes with ``sqlite3.ProgrammingError: SQLite objects created in a - thread can only be used in that same thread``. - """ - from pathlib import Path as _Path - - if not isinstance(path, _Path) or not path.exists(): - return None - conn = self._conns.get(name) - if conn is None: - try: - conn = sqlite3.connect( - f"file:{path}?mode=ro", - uri=True, - check_same_thread=False, - ) - self._conns[name] = conn - except Exception as exc: - _log.warning("[metrics] failed to open %s: %s", name, exc) - return None - return conn - - def _close_conn(self, name: str) -> None: - """Close and evict a connection (called on error to force re-open next scrape).""" - conn = self._conns.pop(name, None) - if conn is not None: - try: - conn.close() - except Exception: - pass - def collect(self): # type: ignore[override] # Lazy imports — keep metrics.py importable from tests without the # full DB modules loaded. @@ -263,7 +209,7 @@ def collect(self): # type: ignore[override] ) g_raids = GaugeMetricFamily( "raid_encounters_total", - "Curated raid-encounter strategy rows in raids.db", + "Curated raid-encounter strategy rows in the raids schema", ) g_triggers = GaugeMetricFamily( "act_triggers_total", @@ -303,19 +249,18 @@ def collect(self): # type: ignore[override] except Exception: _log.exception("[metrics] parses-schema collector error") - # raids.db — strategies + the ACT trigger pack. - conn = self._get_conn("raids", raids_db.DB_PATH) - if conn is not None: - try: + # raids schema (Postgres) — strategies + the ACT trigger pack. + try: + with pg.connection() as conn: + conn.execute(pg.search_path_sql(raids_db.SCHEMA)) row = conn.execute(_SQL["count_raid_encounters"]).fetchone() - g_raids.add_metric([], row[0] if row else 0) + g_raids.add_metric([], row["n"] if row else 0) row = conn.execute(_SQL["count_act_triggers"]).fetchone() - g_triggers.add_metric([], row[0] if row else 0) + g_triggers.add_metric([], row["n"] if row else 0) row = conn.execute(_SQL["count_act_spell_timers"]).fetchone() - g_spell_timers.add_metric([], row[0] if row else 0) - except Exception: - _log.exception("[metrics] raids.db collector error") - self._close_conn("raids") + g_spell_timers.add_metric([], row["n"] if row else 0) + except Exception: + _log.exception("[metrics] raids-schema collector error") yield g_users yield g_claims @@ -338,17 +283,13 @@ class _DBFileSizeCollector(Collector): def collect(self): # type: ignore[override] from backend.eq2db import classes as classes_db from backend.eq2db import items as items_db - from backend.eq2db import raids as raids_db from backend.eq2db import recipes as recipes_db from backend.eq2db import spells as spells_db - from backend.eq2db import zones as zones_db # Map label → Path. Centralised so adding a new DB is one tuple. - # (users + parses moved to Postgres — covered by the schema-size - # gauge below instead of a file stat.) + # (users + parses + census + zones + raids moved to Postgres — + # covered by the schema-size gauge below instead of a file stat.) candidates = [ - ("raids", raids_db.DB_PATH), - ("zones", zones_db.DB_PATH), ("items", items_db.DB_PATH), ("spells", spells_db.DB_PATH), ("recipes", recipes_db.DB_PATH), diff --git a/backend/server/metrics.sql b/backend/server/metrics.sql index bee27aca..96629983 100644 --- a/backend/server/metrics.sql +++ b/backend/server/metrics.sql @@ -13,10 +13,10 @@ SELECT COUNT(*) AS n FROM encounters WHERE hidden_at IS NULL; SELECT COUNT(*) AS n FROM encounters WHERE hidden_at IS NOT NULL; -- :name count_raid_encounters -SELECT COUNT(*) FROM raid_encounters; +SELECT COUNT(*) AS n FROM raid_encounters; -- :name count_act_triggers -SELECT COUNT(*) FROM act_triggers; +SELECT COUNT(*) AS n FROM act_triggers; -- :name count_act_spell_timers -SELECT COUNT(*) FROM act_spell_timers; +SELECT COUNT(*) AS n FROM act_spell_timers; diff --git a/db/migrations/0004_zones.sql b/db/migrations/0004_zones.sql index 019162d0..5b13da6d 100644 --- a/db/migrations/0004_zones.sql +++ b/db/migrations/0004_zones.sql @@ -85,9 +85,14 @@ CREATE TABLE featured_raid_expansions ( added_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) ); +-- position + category were post-launch SQLite ALTERs (drag-reorder lanes on +-- /raids) — folded into the base table here. category NULL = the implicit +-- Uncategorised lane (listing sorts it first via NULLS FIRST). CREATE TABLE featured_raid_zones ( - zone_id bigint PRIMARY KEY REFERENCES zones(id) ON DELETE CASCADE, - added_at bigint NOT NULL DEFAULT floor(extract(epoch from now())) + zone_id bigint PRIMARY KEY REFERENCES zones(id) ON DELETE CASCADE, + added_at bigint NOT NULL DEFAULT floor(extract(epoch from now())), + position integer NOT NULL DEFAULT 0, + category text ); CREATE TABLE featured_raid_categories ( diff --git a/scripts/build_zones_db.py b/scripts/build_zones_db.py index 48589fdb..ed87793b 100644 --- a/scripts/build_zones_db.py +++ b/scripts/build_zones_db.py @@ -1,19 +1,25 @@ """ -Build ``data/zones/zones.db`` from ``scripts/dev/eq2_zones.cleaned.json``. +Load zone metadata from ``scripts/dev/eq2_zones.cleaned.json`` into the +Postgres ``zones`` schema (zones / zone_types / zone_aliases). Idempotent — re-run after editing the source file. Each upsert replaces the -zone row + its types + its aliases atomically. +zone row + its types + its aliases; zone ids are preserved across rebuilds +(builder-assigned: existing name → existing id, new name → max+1), which is +what keeps the curator-managed boss rosters attached. Zones that vanished +from the source JSON are pruned (and their curator data cascades) — the +prune count is printed so an unexpected removal is visible. -Stamps these ``_meta`` keys on every build so the DB is self-describing: +Stamps these ``_meta`` keys on every build so the schema is self-describing: * ``built_at`` — ISO-8601 UTC timestamp * ``built_from`` — path of the cleaned JSON consumed -Boss rosters (``zone_encounters`` / ``zone_encounter_mobs``) are **not** -written by this script — they are web-editable by admins and contributors -via the per-zone editor in the raids UI. +Boss rosters (``zone_encounters`` / ``zone_encounter_mobs``) and the +``featured_*`` curation tables are **not** written by this script — they are +web-editable by admins and contributors via the per-zone editor in the +raids UI, and a metadata rebuild never touches them. -Usage: +Usage (needs DATABASE_URL / SUPABASE_DB_URL in the environment): .venv/Scripts/python scripts/build_zones_db.py .venv/Scripts/python scripts/build_zones_db.py --source path/to/other.json @@ -63,18 +69,18 @@ def _load_dungeons_into_db(path: Path, conn) -> tuple[int, list[str]]: for _expansion, names in by_expansion.items(): for name in names: row = conn.execute( - "SELECT id FROM zones WHERE name = ?", + "SELECT id FROM zones WHERE name = %s", (name,), ).fetchone() if row is None: unmatched.append(name) continue - zone_id = row[0] - # INSERT OR IGNORE so re-running the builder is idempotent — + zone_id = row["id"] + # ON CONFLICT DO NOTHING so re-running the builder is idempotent — # PRIMARY KEY (zone_id, type) prevents duplicates anyway, but # being explicit keeps the intent clear. conn.execute( - "INSERT OR IGNORE INTO zone_types (zone_id, type) VALUES (?, ?)", + "INSERT INTO zone_types (zone_id, type) VALUES (%s, %s) ON CONFLICT DO NOTHING", (zone_id, "dungeon"), ) rows_added += 1 @@ -96,10 +102,9 @@ def main() -> int: help=f"Cleaned-JSON zones file to load (default: {DEFAULT_SOURCE})", ) parser.add_argument( - "--db", - type=Path, - default=zones_db.DB_PATH, - help=f"SQLite output path (default: {zones_db.DB_PATH})", + "--schema", + default=zones_db.SCHEMA, + help=f"Target Postgres schema (default: {zones_db.SCHEMA!r}; override for scratch loads).", ) parser.add_argument( "--dungeons", @@ -126,12 +131,17 @@ def main() -> int: return 1 print(f"Loading {len(zones)} zones from {args.source.name}") - print(f"Target DB: {args.db}") + print(f"Target schema: {args.schema}") - cat = zones_db.ZoneCatalogue(args.db) + cat = zones_db.ZoneCatalogue(args.schema) conn = cat.init_db() try: n = cat.upsert_zones(zones, conn) + # Prune zones that no longer exist in the source. This CASCADEs the + # pruned zones' types/aliases AND any curator boss rosters attached + # to them — correct for a genuinely-removed zone, but always report + # the count so an unexpected prune is caught at the console. + removed = cat.delete_zones_not_named(conn, [z["name"] for z in zones]) zones_db.set_meta(conn, "built_at", dt.datetime.now(dt.timezone.utc).isoformat()) zones_db.set_meta(conn, "built_from", str(args.source)) zones_db.set_meta(conn, "source_count", str(len(zones))) @@ -149,10 +159,12 @@ def main() -> int: conn.close() print(f"Upserted {n} zones.") + if removed: + print(f"PRUNED {removed} zone(s) no longer present in the source JSON (curator rosters on them cascaded).") if dungeons_added: print(f"Tagged {dungeons_added} zones as 'dungeon' from {args.dungeons.name}") if dungeons_unmatched: - print(f"WARN: {len(dungeons_unmatched)} dungeon names not found in zones.db (typos?):") + print(f"WARN: {len(dungeons_unmatched)} dungeon names not found in the zones schema (typos?):") for name in dungeons_unmatched[:5]: print(f" - {name}") if len(dungeons_unmatched) > 5: diff --git a/scripts/dev/_smoke_test_zones_db.py b/scripts/dev/_smoke_test_zones_db.py index 9c566e93..d1fcf99a 100644 --- a/scripts/dev/_smoke_test_zones_db.py +++ b/scripts/dev/_smoke_test_zones_db.py @@ -1,9 +1,9 @@ -"""Smoke tests for data/zones/zones.db — driven via the public -census.zones_db API + raw SQL for integrity checks.""" +"""Smoke tests for the Postgres `zones` schema — driven via the public +zones_db catalogue API + raw SQL for integrity checks. Needs DATABASE_URL +(or SUPABASE_DB_URL) in the environment.""" from __future__ import annotations -import sqlite3 import sys from pathlib import Path @@ -14,7 +14,6 @@ from backend.eq2db import zones as zones_db # noqa: E402 -DB = zones_db.DB_PATH failed: list[str] = [] @@ -25,14 +24,19 @@ def check(label: str, cond: bool, detail: str = "") -> None: failed.append(label) +def _val(conn, sql: str): + """First column of the first row — positional-fetch shim for the raw + integrity queries (psycopg rows are dicts).""" + row = conn.execute(sql).fetchone() + return next(iter(row.values())) if row else None + + print("=" * 70) -print(f"DB SMOKE: {DB}") +print(f"DB SMOKE: zones schema {zones_db.catalogue.schema!r}") print("=" * 70) -check("DB file exists", DB.exists(), str(DB)) - -with sqlite3.connect(DB) as conn: - conn.row_factory = sqlite3.Row +conn = zones_db.catalogue.init_db() +try: # ── _meta provenance keys ───────────────────────────────────────── print("\n--- META ---") for key in ("built_at", "built_from", "source_count"): @@ -41,21 +45,21 @@ def check(label: str, cond: bool, detail: str = "") -> None: # ── Counts ──────────────────────────────────────────────────────── print("\n--- COUNTS ---") - n_zones = conn.execute("SELECT COUNT(*) FROM zones").fetchone()[0] - n_types = conn.execute("SELECT COUNT(*) FROM zone_types").fetchone()[0] - n_aliases = conn.execute("SELECT COUNT(*) FROM zone_aliases").fetchone()[0] - check(f"zones table has 1124 rows (matches cleaned JSON)", n_zones == 1124, f"got {n_zones}") - check(f"zone_types non-empty", n_types > 1000, f"got {n_types}") - check(f"zone_aliases populated from merge file", n_aliases >= 2, f"got {n_aliases}") + n_zones = _val(conn, "SELECT COUNT(*) FROM zones") + n_types = _val(conn, "SELECT COUNT(*) FROM zone_types") + n_aliases = _val(conn, "SELECT COUNT(*) FROM zone_aliases") + check("zones table has 1124 rows (matches cleaned JSON)", n_zones == 1124, f"got {n_zones}") + check("zone_types non-empty", n_types > 1000, f"got {n_types}") + check("zone_aliases populated from merge file", n_aliases >= 2, f"got {n_aliases}") # ── FK integrity ────────────────────────────────────────────────── print("\n--- FK INTEGRITY ---") - orphan_types = conn.execute( - "SELECT COUNT(*) FROM zone_types t WHERE NOT EXISTS (SELECT 1 FROM zones z WHERE z.id = t.zone_id)" - ).fetchone()[0] - orphan_aliases = conn.execute( - "SELECT COUNT(*) FROM zone_aliases a WHERE NOT EXISTS (SELECT 1 FROM zones z WHERE z.id = a.zone_id)" - ).fetchone()[0] + orphan_types = _val( + conn, "SELECT COUNT(*) FROM zone_types t WHERE NOT EXISTS (SELECT 1 FROM zones z WHERE z.id = t.zone_id)" + ) + orphan_aliases = _val( + conn, "SELECT COUNT(*) FROM zone_aliases a WHERE NOT EXISTS (SELECT 1 FROM zones z WHERE z.id = a.zone_id)" + ) check("zone_types has no orphan zone_ids", orphan_types == 0, f"{orphan_types} orphans") check("zone_aliases has no orphan zone_ids", orphan_aliases == 0, f"{orphan_aliases} orphans") @@ -72,10 +76,16 @@ def check(label: str, cond: bool, detail: str = "") -> None: "idx_zone_aliases_zone", } actual = { - r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'") + r["indexname"] + for r in conn.execute( + "SELECT indexname FROM pg_indexes WHERE schemaname = %s", + (zones_db.catalogue.schema,), + ).fetchall() } missing = expected_indexes - actual check("All expected indexes exist", not missing, f"missing: {missing}" if missing else "") +finally: + conn.close() # ── find_by_name: canonical match ───────────────────────────────────── @@ -187,21 +197,29 @@ def check(label: str, cond: bool, detail: str = "") -> None: # Curated from scripts/dev/eq2_raid_bosses.review.txt — EoF + RoK only # in current scope. 14 zones, 55 encounters as of the curator pass. print("\n--- zone_encounters / zone_encounter_mobs ---") -with sqlite3.connect(DB) as conn: - n_enc = conn.execute("SELECT COUNT(*) FROM zone_encounters").fetchone()[0] - n_mobs = conn.execute("SELECT COUNT(*) FROM zone_encounter_mobs").fetchone()[0] +conn = zones_db.catalogue.init_db() +try: + n_enc = _val(conn, "SELECT COUNT(*) FROM zone_encounters") + n_mobs = _val(conn, "SELECT COUNT(*) FROM zone_encounter_mobs") +finally: + conn.close() print(f" zone_encounters rows: {n_enc} zone_encounter_mobs rows: {n_mobs}") if n_enc > 0: # FK integrity - with sqlite3.connect(DB) as conn: - orphan_enc = conn.execute( - "SELECT COUNT(*) FROM zone_encounters e WHERE NOT EXISTS (SELECT 1 FROM zones z WHERE z.id = e.zone_id)" - ).fetchone()[0] - orphan_mobs = conn.execute( + conn = zones_db.catalogue.init_db() + try: + orphan_enc = _val( + conn, + "SELECT COUNT(*) FROM zone_encounters e WHERE NOT EXISTS (SELECT 1 FROM zones z WHERE z.id = e.zone_id)", + ) + orphan_mobs = _val( + conn, "SELECT COUNT(*) FROM zone_encounter_mobs m " - "WHERE NOT EXISTS (SELECT 1 FROM zone_encounters e WHERE e.id = m.encounter_id)" - ).fetchone()[0] + "WHERE NOT EXISTS (SELECT 1 FROM zone_encounters e WHERE e.id = m.encounter_id)", + ) + finally: + conn.close() check("zone_encounters has no orphan zone_ids", orphan_enc == 0, f"{orphan_enc} orphans") check("zone_encounter_mobs has no orphan encounter_ids", orphan_mobs == 0, f"{orphan_mobs} orphans") check("every encounter has at least one mob", n_mobs >= n_enc, f"{n_enc} enc, {n_mobs} mobs") @@ -289,9 +307,12 @@ def check(label: str, cond: bool, detail: str = "") -> None: ) # _meta bookkeeping - with sqlite3.connect(DB) as conn: + conn = zones_db.catalogue.init_db() + try: bz = zones_db.get_meta(conn, "bosses_zones") bt = zones_db.get_meta(conn, "bosses_total") + finally: + conn.close() check("bosses_zones meta populated", bz is not None and int(bz) > 0, f"got {bz}") check( "bosses_total meta matches actual encounter count", diff --git a/scripts/dev/apply_polish.py b/scripts/dev/apply_polish.py index 551435dd..f240efb8 100644 --- a/scripts/dev/apply_polish.py +++ b/scripts/dev/apply_polish.py @@ -26,7 +26,6 @@ import argparse import json -import sqlite3 import sys import time from pathlib import Path @@ -63,23 +62,22 @@ def apply_entries(entries: list[dict], *, dry_run: bool = False) -> dict: conn = raids_db.init_db() try: - conn.row_factory = sqlite3.Row for entry in entries: polished = (entry.get("polished_md") or "").strip() zone_name = entry["zone_name"] mob_name = entry["mob_name"] zrow = conn.execute( - "SELECT id FROM raid_zones WHERE zone_name_lower = ?", + "SELECT id FROM raid_zones WHERE zone_name_lower = %s", (zone_name.lower(),), ).fetchone() if zrow is None: - print(f" [warn] zone not in raids.db: {zone_name!r}") + print(f" [warn] zone not in the raids schema: {zone_name!r}") n_zones_unknown += 1 continue erow = conn.execute( - "SELECT id, mob_name FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?", + "SELECT id, mob_name FROM raid_encounters WHERE raid_zone_id = %s AND mob_name_lower = %s", (zrow["id"], mob_name.lower()), ).fetchone() if erow is None: @@ -101,10 +99,10 @@ def apply_entries(entries: list[dict], *, dry_run: bool = False) -> dict: """ UPDATE raid_encounters SET strategy_md = NULL, - source = ?, - last_edited_at = ?, - last_edited_by = ? - WHERE id = ? + source = %s, + last_edited_at = %s, + last_edited_by = %s + WHERE id = %s """, (raids_db.SOURCE_MANUAL, now, "ai-polish", erow["id"]), ) diff --git a/scripts/dev/apply_rebalance.py b/scripts/dev/apply_rebalance.py index 33bc16f6..45c530a9 100644 --- a/scripts/dev/apply_rebalance.py +++ b/scripts/dev/apply_rebalance.py @@ -27,7 +27,6 @@ import argparse import json -import sqlite3 import sys import time from pathlib import Path @@ -65,15 +64,14 @@ def apply_zones(zones: list[dict], *, dry_run: bool = False) -> dict: conn = raids_db.init_db() try: - conn.row_factory = sqlite3.Row for zone in zones: zone_name = zone["zone_name"] zrow = conn.execute( - "SELECT id, overview_md FROM raid_zones WHERE zone_name_lower = ?", + "SELECT id, overview_md FROM raid_zones WHERE zone_name_lower = %s", (zone_name.lower(),), ).fetchone() if zrow is None: - print(f" [warn] zone not in raids.db: {zone_name!r}") + print(f" [warn] zone not in the raids schema: {zone_name!r}") n_zones_unknown += 1 continue @@ -89,11 +87,11 @@ def apply_zones(zones: list[dict], *, dry_run: bool = False) -> dict: conn.execute( """ UPDATE raid_zones SET - overview_md = ?, - source = ?, - last_edited_at = ?, - last_edited_by = ? - WHERE id = ? + overview_md = %s, + source = %s, + last_edited_at = %s, + last_edited_by = %s + WHERE id = %s """, (new_overview, raids_db.SOURCE_MANUAL, now, "ai-rebalance", zrow["id"]), ) @@ -109,7 +107,7 @@ def apply_zones(zones: list[dict], *, dry_run: bool = False) -> dict: mob_name = enc["mob_name"] erow = conn.execute( "SELECT id, mob_name, strategy_md FROM raid_encounters " - "WHERE raid_zone_id = ? AND mob_name_lower = ?", + "WHERE raid_zone_id = %s AND mob_name_lower = %s", (zrow["id"], mob_name.lower()), ).fetchone() if erow is None: @@ -131,11 +129,11 @@ def apply_zones(zones: list[dict], *, dry_run: bool = False) -> dict: conn.execute( """ UPDATE raid_encounters SET - strategy_md = ?, - source = ?, - last_edited_at = ?, - last_edited_by = ? - WHERE id = ? + strategy_md = %s, + source = %s, + last_edited_at = %s, + last_edited_by = %s + WHERE id = %s """, (new_strategy, raids_db.SOURCE_MANUAL, now, "ai-rebalance", erow["id"]), ) diff --git a/scripts/dev/apply_zone_overviews.py b/scripts/dev/apply_zone_overviews.py index d0a2aa79..2af86446 100644 --- a/scripts/dev/apply_zone_overviews.py +++ b/scripts/dev/apply_zone_overviews.py @@ -18,7 +18,6 @@ import argparse import json -import sqlite3 import sys import time from pathlib import Path @@ -48,17 +47,16 @@ def apply_entries(entries: list[dict], *, dry_run: bool = False) -> dict: conn = raids_db.init_db() try: - conn.row_factory = sqlite3.Row for entry in entries: zone_name = entry["zone_name"] cleaned = (entry.get("cleaned_md") or "").strip() zrow = conn.execute( - "SELECT id, overview_md FROM raid_zones WHERE zone_name_lower = ?", + "SELECT id, overview_md FROM raid_zones WHERE zone_name_lower = %s", (zone_name.lower(),), ).fetchone() if zrow is None: - print(f" [warn] zone not in raids.db: {zone_name!r}") + print(f" [warn] zone not in the raids schema: {zone_name!r}") n_zones_unknown += 1 continue @@ -76,11 +74,11 @@ def apply_entries(entries: list[dict], *, dry_run: bool = False) -> dict: conn.execute( """ UPDATE raid_zones SET - overview_md = ?, - source = ?, - last_edited_at = ?, - last_edited_by = ? - WHERE id = ? + overview_md = %s, + source = %s, + last_edited_at = %s, + last_edited_by = %s + WHERE id = %s """, (new_value, raids_db.SOURCE_MANUAL, now, "ai-audit", zrow["id"]), ) diff --git a/scripts/dev/export_for_polish.py b/scripts/dev/export_for_polish.py index 9ae7a6ee..8adf51bb 100644 --- a/scripts/dev/export_for_polish.py +++ b/scripts/dev/export_for_polish.py @@ -29,7 +29,6 @@ import argparse import json -import sqlite3 import sys from pathlib import Path @@ -43,10 +42,9 @@ def _load_entries() -> list[dict]: """Pull every (zone, mob, current_md) with non-empty strategy from - raids.db, sorted by zone then position for stable ordering.""" - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row - rows = conn.execute( + the raids schema, sorted by zone then position for stable ordering.""" + with raids_db.init_db() as conn: + return conn.execute( """ SELECT z.zone_name, e.mob_name, e.position, e.wiki_url, e.strategy_md, e.source FROM raid_encounters e @@ -55,7 +53,6 @@ def _load_entries() -> list[dict]: ORDER BY z.zone_name, e.position, e.mob_name """ ).fetchall() - return [dict(r) for r in rows] def _balanced_chunks(entries: list[dict], n: int, body_key: str = "current_md") -> list[list[dict]]: diff --git a/scripts/dev/export_for_rebalance.py b/scripts/dev/export_for_rebalance.py index 5608aaac..658ca353 100644 --- a/scripts/dev/export_for_rebalance.py +++ b/scripts/dev/export_for_rebalance.py @@ -56,7 +56,6 @@ import argparse import json -import sqlite3 import sys from pathlib import Path @@ -80,8 +79,7 @@ def _zone_payload_size(z: dict) -> int: def _load_zones() -> list[dict]: """Every zone that has at least one encounter with strategy content OR a non-empty overview — anything with no content has nothing to rebalance.""" - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: zone_rows = conn.execute( """ SELECT z.id, z.zone_name, z.expansion_short, z.overview_md AS current_overview_md @@ -103,7 +101,7 @@ def _load_zones() -> list[dict]: """ SELECT mob_name, position, wiki_url, strategy_md AS current_strategy_md FROM raid_encounters - WHERE raid_zone_id = ? + WHERE raid_zone_id = %s AND strategy_md IS NOT NULL AND TRIM(strategy_md) != '' ORDER BY position, mob_name diff --git a/scripts/dev/export_zone_overviews.py b/scripts/dev/export_zone_overviews.py index f0106fc2..2c896aa9 100644 --- a/scripts/dev/export_zone_overviews.py +++ b/scripts/dev/export_zone_overviews.py @@ -21,7 +21,6 @@ import argparse import json -import sqlite3 import sys from pathlib import Path @@ -49,25 +48,22 @@ def _balanced_chunks(entries: list[dict], n: int) -> list[list[dict]]: def _load_entries() -> list[dict]: """Every raid_zones row with non-empty overview_md.""" - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row - rows = conn.execute( + with raids_db.init_db() as conn: + return conn.execute( """ SELECT zone_name, expansion_short, source, overview_md AS current_md, -- Bosses list joined-in so the agent can see which mobs -- belong to the zone and recognise per-boss content to -- strip out. - (SELECT GROUP_CONCAT(e.mob_name, ', ') + (SELECT string_agg(e.mob_name, ', ' ORDER BY e.position) FROM raid_encounters e - WHERE e.raid_zone_id = raid_zones.id - ORDER BY e.position) AS encounters + WHERE e.raid_zone_id = raid_zones.id) AS encounters FROM raid_zones WHERE overview_md IS NOT NULL AND TRIM(overview_md) != '' ORDER BY zone_name """ ).fetchall() - return [dict(r) for r in rows] def main(argv: list[str] | None = None) -> int: diff --git a/scripts/dev/ingest_act_triggers.py b/scripts/dev/ingest_act_triggers.py index d6cf94f8..a8b01fe1 100644 --- a/scripts/dev/ingest_act_triggers.py +++ b/scripts/dev/ingest_act_triggers.py @@ -141,8 +141,6 @@ def resolve_encounter(zone_name: str, position: int) -> tuple[int, str] | None: ``zone_name`` must match canonical zones_db naming; lookups go through ``find_by_name`` which respects aliases.""" - import sqlite3 - z = zones_db.catalogue.find_by_name(zone_name) if z is None: return None @@ -157,11 +155,9 @@ def resolve_encounter(zone_name: str, position: int) -> tuple[int, str] | None: if mob_name is None: return None - raids_db.init_db().close() - with sqlite3.connect(raids_db.path) as conn: - conn.row_factory = sqlite3.Row + with raids_db.init_db() as conn: zrow = conn.execute( - "SELECT id FROM raid_zones WHERE zone_name_lower = ?", + "SELECT id FROM raid_zones WHERE zone_name_lower = %s", (canonical_zone.lower(),), ).fetchone() if zrow is None: @@ -175,7 +171,7 @@ def resolve_encounter(zone_name: str, position: int) -> tuple[int, str] | None: zone_id = zrow["id"] erow = conn.execute( - "SELECT id FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?", + "SELECT id FROM raid_encounters WHERE raid_zone_id = %s AND mob_name_lower = %s", (zone_id, mob_name.lower()), ).fetchone() if erow is None: @@ -202,13 +198,9 @@ def _trigger_exists(encounter_id: int, regex: str, sound_data: str) -> bool: """Idempotency check — (encounter_id, regex, sound_data) is treated as identity. There's no UNIQUE constraint at the DB level (regexes can legitimately overlap with different sounds), so we check before insert.""" - import sqlite3 - - if not raids_db.path.exists(): - return False - with sqlite3.connect(raids_db.path) as conn: + with raids_db.init_db() as conn: row = conn.execute( - "SELECT id FROM act_triggers WHERE raid_encounter_id = ? AND regex = ? AND sound_data = ?", + "SELECT id FROM act_triggers WHERE raid_encounter_id = %s AND regex = %s AND sound_data = %s", (encounter_id, regex, sound_data), ).fetchone() return row is not None @@ -304,14 +296,11 @@ def apply_ingest(*, xml_path: Path, dry_run: bool) -> dict: try: # Find existing by (encounter_id, name_lower) so re-runs update, # don't 409. - import sqlite3 as _sq - - with _sq.connect(raids_db.path) as q: - row = q.execute( - "SELECT id FROM act_spell_timers WHERE raid_encounter_id = ? AND name_lower = ?", - (encounter_id, spell["name"].lower()), - ).fetchone() - timer_id = row[0] if row else None + row = conn.execute( + "SELECT id FROM act_spell_timers WHERE raid_encounter_id = %s AND name_lower = %s", + (encounter_id, spell["name"].lower()), + ).fetchone() + timer_id = row["id"] if row else None raids_db.upsert_act_spell_timer( conn, timer_id=timer_id, @@ -377,7 +366,7 @@ def main() -> None: mode = "APPLY" if args.apply else "DRY-RUN" print(f"=== ingest_act_triggers ({mode}) ===") print(f" source: {args.xml}") - print(f" raids.db: {raids_db.path}") + print(f" raids schema: {raids_db.schema}") print() summary = apply_ingest(xml_path=args.xml, dry_run=not args.apply) diff --git a/scripts/dev/seed_fake_parses.py b/scripts/dev/seed_fake_parses.py index 547d7c54..ea6e4ed4 100644 --- a/scripts/dev/seed_fake_parses.py +++ b/scripts/dev/seed_fake_parses.py @@ -28,7 +28,6 @@ import argparse import os import random -import sqlite3 import sys import time import uuid @@ -73,24 +72,28 @@ def _curated_bosses(limit: int) -> list[tuple[str, str]]: - """(zone, mob_title) tuples from zones.db curated encounters. Falls back - to heuristic-friendly capitalised fakes if the local db has no curation.""" - conn = sqlite3.connect(f"file:{zones_db.path}?mode=ro", uri=True) + """(zone, mob_title) tuples from the zones schema's curated encounters. + Falls back to heuristic-friendly capitalised fakes if there is no + curation. One mob per encounter (DISTINCT ON e.id — the old SQLite + version's loose GROUP BY picked an arbitrary mob the same way).""" + conn = zones_db.init_db() try: rows = conn.execute( - """SELECT z.name, m.mob_name_lower - FROM zone_encounter_mobs m - JOIN zone_encounters e ON e.id = m.encounter_id - JOIN zones z ON z.id = e.zone_id - WHERE length(m.mob_name_lower) >= 6 - GROUP BY e.id ORDER BY z.name, e.id LIMIT ?""", + """SELECT name, mob_name_lower FROM ( + SELECT DISTINCT ON (e.id) e.id AS eid, z.name AS name, m.mob_name_lower AS mob_name_lower + FROM zone_encounter_mobs m + JOIN zone_encounters e ON e.id = m.encounter_id + JOIN zones z ON z.id = e.zone_id + WHERE length(m.mob_name_lower) >= 6 + ORDER BY e.id, m.position + ) sub ORDER BY name, eid LIMIT %s""", (limit,), ).fetchall() finally: conn.close() if rows: - return [(zone, mob[0].upper() + mob[1:]) for zone, mob in rows] - print("! zones.db has no curated encounters — using heuristic boss names") + return [(r["name"], r["mob_name_lower"][0].upper() + r["mob_name_lower"][1:]) for r in rows] + print("! zones schema has no curated encounters — using heuristic boss names") return [ ("Deathtoll", "Tarinax the Destroyer"), ("Deathtoll", "Xerkizh The Creator"), diff --git a/tests/conftest.py b/tests/conftest.py index 977c07f3..30cff2f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,10 +65,8 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 BE-096: moved from module-level os.environ calls to avoid a race with pytest plugins (e.g. pytest-asyncio) that may import web.app during plugin discovery.""" - os.environ["DB_ZONES_PATH"] = str(_TEST_DB_DIR / "zones.db") os.environ["DB_SPELLS_PATH"] = str(_TEST_DB_DIR / "spells.db") os.environ["DB_RECIPES_PATH"] = str(_TEST_DB_DIR / "recipes.db") - os.environ["DB_RAIDS_PATH"] = str(_TEST_DB_DIR / "raids.db") # DB_CLASSES_PATH intentionally NOT overridden — classes.db is the # committed source-of-truth (data/classes/classes.db) and is read-only # at runtime. Tests read it directly; nothing writes to it. Pointing @@ -114,18 +112,16 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 from backend.db_helpers import resolve_db_path # noqa: PLC0415 from backend.eq2db import classes as classes_db from backend.eq2db import items as items_db - from backend.eq2db import raids as raids_db from backend.eq2db import recipes as recipes_db from backend.eq2db import spells as spells_db - from backend.eq2db import zones as zones_db # eq2db catalogue modules: re-point both the module constant AND the # shared catalogue instance (its path was captured at import time). + # (zones moved to Postgres — its tests lease scratch schemas via + # tests/fixtures/pg instead of a tmp file.) for mod, env_var, subdir, filename in ( - (zones_db, "DB_ZONES_PATH", "zones", "zones.db"), (spells_db, "DB_SPELLS_PATH", "spells", "spells.db"), (recipes_db, "DB_RECIPES_PATH", "recipes", "recipes.db"), - (raids_db, "DB_RAIDS_PATH", "raids", "raids.db"), (items_db, "DB_ITEMS_PATH", "items", "items.db"), (classes_db, "DB_CLASSES_PATH", "classes", "classes.db"), ): From 4a27db673c43a7a9b59ef01d87b048dbd0b42d61 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 17:44:52 +0100 Subject: [PATCH 25/32] docs(pg): CLAUDE.md reflects the Postgres architecture; zones/raids test fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Key-files table, env vars (DATABASE_URL chain, retired DB_*_PATH rows, R2 as Actions secrets), backups (nightly pg_dump + restore drill), zone-refresh and raid-seed workflows (script runs against DATABASE_URL — the manual volume uploads are gone), erasure/cleanup/store rows updated. tests/fixtures/zones_raids_db.py: leased zones_schema/raids_schema fixtures, re-exported via conftest. app.py: the startup parses checkout closes (pooled-slot leak the zones agent caught). Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 89 ++++++++++++-------------------- tests/conftest.py | 1 + tests/fixtures/zones_raids_db.py | 44 ++++++++++++++++ 3 files changed, 79 insertions(+), 55 deletions(-) create mode 100644 tests/fixtures/zones_raids_db.py diff --git a/CLAUDE.md b/CLAUDE.md index 85ec546d..d110b9bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,17 +12,17 @@ A web companion site for EverQuest 2 (TLE), with a Discord bot for spot checks ( | `backend/census/models.py` | Dataclasses: `ItemData`, `ItemStat`, `ItemEffect`, `GuildData`, `GuildMember`, `CharacterSpells`, `SpellEntry`, `NodeAA`, `CharacterAAs` | | `backend/census/constants.py` | `STAT_MAP` (stat display names/groups), class frozensets (`FIGHTERS`, `PRIESTS`, `SCOUTS`, `MAGES`, `ARTISANS`), `ARCHETYPES`, `CLASS_GROUPS`, `TYPEINFO_DISPLAY`, `ITEM_DISPLAY` | | `backend/census/item_parser.py` | Item data parsing helpers (parse_item, parse_stats, parse_effects, parse_flags, etc.) extracted from client.py | -| `backend/eq2db/spells.py` | Local SQLite spell catalogue — `SpellCatalogue` class (shared `catalogue` instance): strip_roman, unique_highest_entries, load_blocklist, find_by_ids, find_by_crc, spell_to_row, upsert_spells. Every eq2db module follows this convention: pure helpers are staticmethods with full bodies in the class, DB reads are instance methods on a per-instance `path`, tests construct `XCatalogue(tmp_db)` and patch the instance, `tests/conftest.py` re-points `catalogue.path`. All seven catalogues inherit `BaseCatalogue` (`backend/db_catalogue.py`) — `init_db` is a template method (connection preamble + commit in the base); subclasses implement `_create_schema` (+ optional `_post_init` backfills), set `FOREIGN_KEYS`/`CREATE_META` class flags, and override `clear_caches` when they hold caches. The same architecture covers the other SQLite stores: `CensusStore` (census/store.py), `ParsesStore` (server/parses/db.py), and the seven users.db domain stores (`XStore(AsyncStoreBase)` in server/db/ — schema stays with the package init_db orchestrator; the `backend.server.db` facade re-exports bound methods so `users_db.fn(...)` call sites are unchanged; conftest re-points every store via `users_db.ALL_STORES`). | +| `backend/eq2db/spells.py` | Local SQLite spell catalogue — `SpellCatalogue` class (shared `catalogue` instance): strip_roman, unique_highest_entries, load_blocklist, find_by_ids, find_by_crc, spell_to_row, upsert_spells. Every eq2db module follows this convention: pure helpers are staticmethods with full bodies in the class, DB reads are instance methods on a per-instance `path`, tests construct `XCatalogue(tmp_db)` and patch the instance, `tests/conftest.py` re-points `catalogue.path`. All seven catalogues inherit `BaseCatalogue` (`backend/db_catalogue.py`) — `init_db` is a template method (connection preamble + commit in the base); subclasses implement `_create_schema` (+ optional `_post_init` backfills), set `FOREIGN_KEYS`/`CREATE_META` class flags, and override `clear_caches` when they hold caches. The WRITABLE families (users / parses / census / zones / raids) live in **Supabase Postgres** — one database, one schema per family. DDL is owned by `db/migrations/NNNN_.sql`, applied by `backend/pg_migrate.py` from the app lifespan (deploy = migrate; ledger `public.schema_migrations` + advisory lock). Connections come from `backend/pg.py` (lifespan-owned psycopg pools, pool-if-open-else-direct; schema selected via `SET search_path` at checkout so all SQL stays unqualified; `%s` params, dict rows). The users domain stores are `XStore(PgStoreBase)`; `CensusStore` / `ParsesStore` / `ZoneCatalogue` / `RaidCatalogue` are `PgCatalogue` — `init_db()` KEEPS ITS NAME but returns a pooled schema-scoped connection proxy (close() returns to pool). The `backend.server.db` facade re-exports are unchanged; tests lease scratch schemas via `tests/fixtures/pg.py` (TEST_DATABASE_URL, local PG17) and re-point `store.schema`. | | `backend/eq2db/recipes.py` | Local SQLite recipe catalogue (~70k rows) — `RecipeCatalogue`: find_by_id, find_by_name, find_by_spell, find_spells_by_tier, find_by_output_id. Secondary components stored as JSON array. | | `backend/eq2db/aas.py` | Local SQLite AA catalogue (aas.db — committed like classes.db). Tables `aa_trees` (tree_type + max_points precomputed at build) + `aa_nodes` (incl. per-node unlock rules: points_to_unlock, classification_points_required, first_parent_id/tier) + `aa_limits` (per-xpac caps + `visible_rows` era node curation from aa_limits.json — pre-SF xpacs hide class rows 5-6 / subclass rows 16+19, verified against live Wuoshi census). Accessors: `load_tree_index`, `tree_node_costs`, `tree_max_points`, `total_max_points`, `get_tree`, `xpac_limits` (short-code alias tolerant; returns aa_cap + unlocked_trees + visible_rows); `detect_tree_type` is build-time only. Rebuild: `scripts/download_aa_trees.py` (tree JSONs are local intermediates, gitignored) then `scripts/build_aas_db.py`; commit the db. Era filtering surfaces via `/api/aa/config`.`visible_rows` → `filterTreeForEra` in CharacterAAsTab. | -| `backend/eq2db/zones.py` | Local SQLite zone catalogue (~1124 rows) — `ZoneCatalogue` over frozen dataclass models (`Zone`, `ZoneEncounter`, `ZoneEncounterMob`, `FeaturedRaid*`). Tables: `zones`, `zone_types`, `zone_aliases`, `zone_encounters`, `zone_encounter_mobs` + the featured-raid trio. Lookups: `find_by_name`, `list_by_expansion`, `list_by_event`, `list_by_type`, `list_bosses_for_zone`, `find_zones_by_boss`. Zone metadata from `scripts/dev/eq2_zones.cleaned.json`; rebuild via `scripts/build_zones_db.py`. Boss data is curator-managed in-place. | +| `backend/eq2db/zones.py` | Zone catalogue (~1124 rows, Postgres `zones` schema) — `ZoneCatalogue` over frozen dataclass models (`Zone`, `ZoneEncounter`, `ZoneEncounterMob`, `FeaturedRaid*`). Tables: `zones`, `zone_types`, `zone_aliases`, `zone_encounters`, `zone_encounter_mobs` + the featured-raid trio. Lookups: `find_by_name`, `list_by_expansion`, `list_by_event`, `list_by_type`, `list_bosses_for_zone`, `find_zones_by_boss`. Zone metadata from `scripts/dev/eq2_zones.cleaned.json`; rebuild via `scripts/build_zones_db.py`. Boss data is curator-managed in-place. | | `backend/census/wikitext_md.py` | MediaWiki wikitext → markdown converter (mwparserfromhell). Handles EQ2i templates, wikilinks, nested lists, headings, bold/italic. Used by the raid scraper. | -| `backend/eq2db/raids.py` | Local SQLite raid-strategy catalogue — `RaidCatalogue`: `raid_zones` + `raid_encounters` (markdown blob per encounter) + `raid_encounter_revisions` + ACT trigger/spell-timer tables (helpers folded in from the former raids_act.py). `SOURCE_*` provenance tokens mirrored as class attributes. `DB_RAIDS_PATH` env var. | -| `backend/census/store.py` | Persistent SQLite store (characters + guilds keyed by name_lower + world) — `CensusStore(BaseCatalogue)`, shared `store` instance (consumers alias it `census_store`). Keep-best-known merge — sparse Census refresh never nulls good data. `DB_CENSUS_PATH` env. Also `guild_history` (one row per guild per UTC day: level/members/accounts/achievements + max-level and distinct-class roster counts) written by `guild_cache._persist_and_publish_guild` via `upsert_guild_history` (prunes past `GUILD_HISTORY_RETENTION_DAYS`=400); served store-only by `GET /api/guild/{name}/history?days=` (never Census; empty list for an unseen guild) → the guild page History tab (`pages/guild/GuildHistoryTab.tsx`, lazy, Recharts in the `vendor-charts` chunk; `components/charts/GuildHistoryChart.tsx` — one axis per chart, fixed colour per series). Chunk naming caveat in `frontend/vite.config.ts`: rolldown puts React's CommonJS body in the alphabetically-first manual chunk, hence `core-react` (check `modulepreload` in dist/index.html after any manualChunks change). | +| `backend/eq2db/raids.py` | Raid-strategy catalogue (Postgres `raids` schema) — `RaidCatalogue`: `raid_zones` + `raid_encounters` (markdown blob per encounter) + `raid_encounter_revisions` + ACT trigger/spell-timer tables (helpers folded in from the former raids_act.py). `SOURCE_*` provenance tokens mirrored as class attributes. | +| `backend/census/store.py` | Persistent store (Postgres `census` schema; characters + guilds keyed by name_lower + world, `data_json` jsonb) — `CensusStore(PgCatalogue)`, shared `store` instance (consumers alias it `census_store`). Keep-best-known merge — sparse Census refresh never nulls good data. Also `guild_history` (one row per guild per UTC day: level/members/accounts/achievements + max-level and distinct-class roster counts) written by `guild_cache._persist_and_publish_guild` via `upsert_guild_history` (prunes past `GUILD_HISTORY_RETENTION_DAYS`=400); served store-only by `GET /api/guild/{name}/history?days=` (never Census; empty list for an unseen guild) → the guild page History tab (`pages/guild/GuildHistoryTab.tsx`, lazy, Recharts in the `vendor-charts` chunk; `components/charts/GuildHistoryChart.tsx` — one axis per chart, fixed colour per series). Chunk naming caveat in `frontend/vite.config.ts`: rolldown puts React's CommonJS body in the alphabetically-first manual chunk, hence `core-react` (check `modulepreload` in dist/index.html after any manualChunks change). | | `backend/image/tooltip.py` | PIL renderer for item tooltips. Renders at 2× then downsamples (SCALE=2, ZOOM=1.3). See file for colour/stat/ordering details. | | `backend/image/aa_tree.py` | AA tree renderers and coordinate systems. See file for tree-type detection and coordinate arithmetic. | -| `backend/bot/bot.py` | Registers all cogs, syncs slash commands to three specific guild IDs (648253204760625160, 955890381847928892, 1502314690041221260) for instant propagation plus a global sync. Also: shared app-command error handler, tracked background-task set cancelled in close(), defensive users_db.init_db() in setup_hook (bot races the web lifespan), and `intents.members = True` — the SERVER MEMBERS privileged intent MUST be toggled in the dev portal or login fails (main.py catches it, logs CRITICAL, web half keeps running). | -| `backend/bot/guild_context.py` | Per-Discord-guild context: `resolve_guild_context(discord_guild_id)` reads the `discord_guild_links` registry (users.db, edited via `/lexicon` link/voice/status/unlink, manage_guild gated) → GuildContext(world, guild_name, voice_channel_id, linked). Unlinked/DM → `FALLBACK_WORLD = "Wuoshi"` (hardcoded by design). All world-aware cogs resolve here — the env WORLD pin is bot-dead. | +| `backend/bot/bot.py` | Registers all cogs, syncs slash commands to three specific guild IDs (648253204760625160, 955890381847928892, 1502314690041221260) for instant propagation plus a global sync. Also: shared app-command error handler, tracked background-task set cancelled in close() (the bot races the web lifespan, but pre-lifespan queries fall back to direct psycopg connections and guild_context degrades on psycopg.Error — no defensive init needed), and `intents.members = True` — the SERVER MEMBERS privileged intent MUST be toggled in the dev portal or login fails (main.py catches it, logs CRITICAL, web half keeps running). | +| `backend/bot/guild_context.py` | Per-Discord-guild context: `resolve_guild_context(discord_guild_id)` reads the `discord_guild_links` registry (users schema, edited via `/lexicon` link/voice/status/unlink, manage_guild gated) → GuildContext(world, guild_name, voice_channel_id, linked). Unlinked/DM → `FALLBACK_WORLD = "Wuoshi"` (hardcoded by design). All world-aware cogs resolve here — the env WORLD pin is bot-dead. | | `backend/bot/render.py` + `messaging.py` | Pure text half of bot responses (table builders, `plan_code_block` → SendPlan for the 2000-char wrap-vs-file decision; fully unit-tested in tests/bot/) + the one discord I/O sender `send_plan`. Cogs are thin adapters. | | `backend/bot/cogs/voice_attendance.py` | Phase 3 voice cross-check: tasks.loop every 120s → for each `/lexicon voice`-configured guild, if `attendance_store.find_live_session(world, guild, now)` (merge-gap window) → snapshot the voice channel members → `record_voice` kind='voice' observations (character_name carries the discord id). Site rollup shows 🎧 + "in voice, not in game" on AWOL. Idle path = 1 SQL per linked guild, no Discord API calls. | | `backend/bot/cogs/items.py` | `/item` — accepts name, numeric ID, or game link | @@ -34,7 +34,7 @@ A web companion site for EverQuest 2 (TLE), with a Discord bot for spot checks ( | `backend/server/api/server.py` | `GET /api/server` — bootstraps the frontend with the active server's world, display name, max_level, current_xpac, launch_dt, and the full public server list | | `backend/server/cache.py` | TTLCache with stale-while-revalidate: character_cache, guild_cache, claim_cache. Character and guild read paths serve from `census_store` first and never block on Census. | | `backend/server/api/aa.py` | GET /api/character/{name}/aas — AA profile list with per-tree data | -| `backend/server/api/aa_plans.py` | AA planner saved builds: CRUD (owner-scoped in SQL) + read-only share by always-minted slug (`GET /api/aa/plan/{slug}`, page `/aa-plan/{slug}`). Store `backend/server/db/aa_plans.py` (users.db). Rule legality is the frontend engine's job (`frontend/src/pages/aaplanner/engine.ts` — point-cost-weighted self-exclusive thresholds, parent ranks, flat 100/tree cap, separate tradeskill pool, no-stranding removals); the server validates structurally. Planner UI = "Planner" mode on the character AA tab (`aaplanner/PlannerMode.tsx`, interactive AATree: click spend / right-click refund). | +| `backend/server/api/aa_plans.py` | AA planner saved builds: CRUD (owner-scoped in SQL) + read-only share by always-minted slug (`GET /api/aa/plan/{slug}`, page `/aa-plan/{slug}`). Store `backend/server/db/aa_plans.py` (users schema). Rule legality is the frontend engine's job (`frontend/src/pages/aaplanner/engine.ts` — point-cost-weighted self-exclusive thresholds, parent ranks, flat 100/tree cap, separate tradeskill pool, no-stranding removals); the server validates structurally. Planner UI = "Planner" mode on the character AA tab (`aaplanner/PlannerMode.tsx`, interactive AATree: click spend / right-click refund). | | `backend/server/api/character/gear_sets.py` | GET /api/character/{name}/gear-sets — saved in-game equipment sets (Census `adventure_sets`), store-first SWR mirroring aa.py; feeds the character-sheet set pills + the compare per-side set picker. Each set carries `stat_deltas` (set − worn, additive stats + active item-set bonuses, from items.db `item_stats`) computed in sibling `stat_deltas.py` — the sheet shows approximated stats when a set is selected | | `backend/server/api/character/rankings.py` | GET /api/character/{name}/rankings — WCL-style per-boss summary over the /api/rankings kills dataset (shared 60s cache). Gated to curated zones.db content only (same universe as the rankings dropdowns — heuristic-matched kills never surface); zone sections carry `expansion` and the response lists the character's `expansions` (newest first) for the tab's xpac dropdown. Class-scoped rank percentiles (best/median vs every same-class parse), All Stars points = 100×best/class-record + rank among class peers (per boss + zone total; peers get credit for bosses the target never killed). Both dps+hps in one response. `zones: []` ⇒ character page hides the Rankings tab (`CharacterRankingsTab.tsx`; percentileColors.ts scale). Fake local data: `scripts/dev/seed_fake_parses.py --character X` (tagged uploaded_by='fake-seed', `--wipe` to remove). | | `backend/server/api/characters.py` | GET /api/characters/search — character name search | @@ -46,12 +46,12 @@ A web companion site for EverQuest 2 (TLE), with a Discord bot for spot checks ( | `backend/server/census_events.py` | In-process async pub/sub backing the SSE stream (single-process only). | | `backend/server/census_refresh.py` | Background refresh orchestration (throttle 15 min / in-flight dedupe / skip-when-down); merges into census_store, updates cache, publishes SSE. `_merge_roster` best-known join. | | `backend/server/api/census.py` | `GET /api/backend/census/health` (first-paint snapshot) + `GET /api/backend/census/stream` (SSE: character/guild refresh records + health changes). | -| `backend/server/parses/cleanup.py` | Parses retention sweep (`run_parse_cleanup`). Trash (non-boss) hard-deleted `PARSE_RETENTION_DAYS` after the fight; named (boss) fights keep only the primary (longest) upload after the same window, deleting duplicate uploads. Reuses `_group_into_fights` so it picks the same primary rankings link to (rankings-safe); never touches soft-deleted rows. Run periodically by `app.py:_parse_cleanup_loop`. | -| `backend/server/api/guild_settings.py` | Per-guild switches, LEADER-or-admin write (Census rank_id 0 — `_leader_chars` / `_leader_chars_cached` in guild.py, the first place rank 0 is distinguished from `_OFFICER_RANKS`), public `GET /api/guild/{name}/settings`, `PUT` audited as `guild_settings_updated`. Store `backend/server/db/guild_settings.py` (`guild_settings` table in users.db, one boolean column per switch, absent row == defaults). Only switch today: `officers_can_delete_parses` (default on) — enforced in `parses/delete.py::_can_delete_encounter` (memoised per guild per request) and `parses/list.py::_compute_permissions` (one IN-query, officers only); gates ONLY officer parse deletion — uploaders keep their own uploads, admins unaffected. `officer-status` also returns `is_leader`; the guild page shows a Settings tab to leader/admin. Added after 2026-09-27. | -| `backend/server/db/erasure.py` | Right-to-erasure (privacy policy §8, 2026-09-28): `erase_user_sync(discord_id)` over BOTH users.db and parses.db — rows that ARE the person are deleted (users, tokens, roles, requests, claims, favourites, downloads, AA plans, availability, voice observations, tamper reports), `*_by` author columns are tombstoned to the placeholder user `deleted`, uploads stay as guild records with `source_dsn` → `plugin:deleted` and `hidden_by` cleared. Routes: admin `DELETE /api/admin/users/{id}` (not self) and self-service `DELETE /api/auth/me` (body `{confirm: }`, clears the session); both audit `user_erased` and clear the claim cache, supporters cache and metrics last-seen map. Privacy page `frontend/src/pages/PrivacyPage.tsx` at `/privacy` is the one route rendered without a login (`PUBLIC_PATHS` + `PublicShell` in App.tsx) — keep its text in step with the code. Companion retention: `VOICE_OBSERVATION_RETENTION_DAYS` (90) swept in `app.py:_parse_cleanup_loop`; `/api/supporters` is session-gated and returns display names; fonts are self-hosted from `frontend/public/fonts` (`src/fonts.css`, regenerate with `scripts/dev/selfhost_fonts.py`) so no visitor IP reaches Google. | -| `backend/server/db/site_settings.py` | Site-wide (not per-server) key/value settings in users.db (`site_settings`), facade aliases `get_site_setting` / `set_site_setting` / `all_site_settings`; absent row == unset. Only key today: `discord_invite_url` — admin `GET`/`PUT /api/admin/site-settings` (regex-validated `https://discord.gg/` or `https://discord.com/invite/`, audited `site_settings_updated`), delivered to the frontend in the `GET /api/server` bootstrap as `discord_invite_url` → `useServer().discordInviteUrl` → `components/DiscordCommunityLink.tsx` (icon in the header's top-right cluster + button on the Support page; renders nothing when unset). Admin UI: `pages/admin/SiteSettingsSection.tsx`. The invite must be made permanent in Discord (Expire after: Never, Max uses: No limit); the site never validates it live. | -| `backend/server/api/guild_recruitment.py` | Guild recruitment API — public read, officer-or-admin write (raid_schedule's `_officer_chars` gate). Profiles keyed by CENSUS GUILD ID in the store (`backend/server/db/guild_recruitment.py`, users.db) so a profile + logo survive guild renames; routes stay name-addressed and every officer save re-resolves the id + refreshes the stored name. Fields: recruiting flag, blocklist-screened description (≤1000), needed classes (validated vs classes.db adventure names), curated tags (`RECRUITMENT_TAGS`), ≤3 roster-validated in-game contacts, `DISCORD_INVITE_RE` link. Logo: base64 JSON upload → PIL re-encode (`_process_logo`: format+dims header-checked pre-decode, exif_transpose, ≤200px WebP q85, ≤128KB) stored as BLOB in users.db (litestream-replicated); served with ETag/304 + nosniff; upload/removal audited with the actor (`guild_logo_uploaded`/`guild_logo_removed` — the abuse trail). `GET /api/recruiting` lists recruiting guilds + member counts via `census_store.latest_guild_member_counts` (guild_history). Daily sweep `backend/server/recruitment_sweep.py` (app lifespan): census lookup BY ID per listed guild — renamed → stored name refreshed, gone → auto-delisted (`recruiting=0`, never deleted), census error → untouched. Erasure tombstones `updated_by`/`logo_uploaded_by`; the logo blob stays (guild asset). Frontend: guild-page Recruitment tab (`pages/guild/GuildRecruitmentTab.tsx`, view for everyone + officer edit) and browse page `pages/RecruitingPage.tsx` at `/recruiting` (nav in BOTH App.tsx BROWSE_ITEMS and MobileNav GROUPS). | -| `backend/server/api/raid_schedule.py` | Guild raid-schedule API. Public `GET /api/guild/{name}/raid-schedule`; officer-or-admin `PUT` (full replace, ≤4 teams × ≤4 raids, each ≤5h, IANA tz, Twitch validated, free text blocklist-screened). Clearing == PUT with empty `teams`. `GET /api/raiding-live` returns the current world's live teams. Tables `raid_teams`/`raid_slots` in `users.db` (`backend/server/db/raid_schedule.py`). | +| `backend/server/parses/cleanup.py` | Parses retention sweeps (`run_parse_cleanup`). Rows: trash (non-boss) hard-deleted `PARSE_RETENTION_DAYS` after the fight; named (boss) fights keep only the primary (longest) upload after the same window. Reuses `_group_into_fights` (rankings-safe); never touches soft-deleted rows. **Detail**: kept fights' `attack_types`+`damage_types` are dropped on a zone-tier schedule (`_classify_zone`: curated raid 30d / curated group-instance 14d / other 7d, env-overridable `PARSE_DETAIL_RETENTION_*_DAYS`); `encounters.detail_pruned_at` is stamped and the parse page shows a "breakdown pruned" notice. encounters+combatants (rankings/list/character pages) live forever. The cutover copy script applies the same tiers at initial load. Run periodically by `app.py:_parse_cleanup_loop`. | +| `backend/server/api/guild_settings.py` | Per-guild switches, LEADER-or-admin write (Census rank_id 0 — `_leader_chars` / `_leader_chars_cached` in guild.py, the first place rank 0 is distinguished from `_OFFICER_RANKS`), public `GET /api/guild/{name}/settings`, `PUT` audited as `guild_settings_updated`. Store `backend/server/db/guild_settings.py` (`guild_settings` table in the users schema, one boolean column per switch, absent row == defaults). Only switch today: `officers_can_delete_parses` (default on) — enforced in `parses/delete.py::_can_delete_encounter` (memoised per guild per request) and `parses/list.py::_compute_permissions` (one IN-query, officers only); gates ONLY officer parse deletion — uploaders keep their own uploads, admins unaffected. `officer-status` also returns `is_leader`; the guild page shows a Settings tab to leader/admin. Added after 2026-09-27. | +| `backend/server/db/erasure.py` | Right-to-erasure (privacy policy §8, 2026-09-28): `erase_user_sync(discord_id)` — ONE Postgres transaction spanning the users and parses schemas (search_path switched mid-transaction; users FKs deferred), so a crash can never leave a half-erased account — rows that ARE the person are deleted (users, tokens, roles, requests, claims, favourites, downloads, AA plans, availability, voice observations, tamper reports), `*_by` author columns are tombstoned to the placeholder user `deleted`, uploads stay as guild records with `source_dsn` → `plugin:deleted` and `hidden_by` cleared. Routes: admin `DELETE /api/admin/users/{id}` (not self) and self-service `DELETE /api/auth/me` (body `{confirm: }`, clears the session); both audit `user_erased` and clear the claim cache, supporters cache and metrics last-seen map. Privacy page `frontend/src/pages/PrivacyPage.tsx` at `/privacy` is the one route rendered without a login (`PUBLIC_PATHS` + `PublicShell` in App.tsx) — keep its text in step with the code. Companion retention: `VOICE_OBSERVATION_RETENTION_DAYS` (90) swept in `app.py:_parse_cleanup_loop`; `/api/supporters` is session-gated and returns display names; fonts are self-hosted from `frontend/public/fonts` (`src/fonts.css`, regenerate with `scripts/dev/selfhost_fonts.py`) so no visitor IP reaches Google. | +| `backend/server/db/site_settings.py` | Site-wide (not per-server) key/value settings in the users schema (`site_settings`), facade aliases `get_site_setting` / `set_site_setting` / `all_site_settings`; absent row == unset. Only key today: `discord_invite_url` — admin `GET`/`PUT /api/admin/site-settings` (regex-validated `https://discord.gg/` or `https://discord.com/invite/`, audited `site_settings_updated`), delivered to the frontend in the `GET /api/server` bootstrap as `discord_invite_url` → `useServer().discordInviteUrl` → `components/DiscordCommunityLink.tsx` (icon in the header's top-right cluster + button on the Support page; renders nothing when unset). Admin UI: `pages/admin/SiteSettingsSection.tsx`. The invite must be made permanent in Discord (Expire after: Never, Max uses: No limit); the site never validates it live. | +| `backend/server/api/guild_recruitment.py` | Guild recruitment API — public read, officer-or-admin write (raid_schedule's `_officer_chars` gate). Profiles keyed by CENSUS GUILD ID in the store (`backend/server/db/guild_recruitment.py`, users schema) so a profile + logo survive guild renames; routes stay name-addressed and every officer save re-resolves the id + refreshes the stored name. Fields: recruiting flag, blocklist-screened description (≤1000), needed classes (validated vs classes.db adventure names), curated tags (`RECRUITMENT_TAGS`), ≤3 roster-validated in-game contacts, `DISCORD_INVITE_RE` link. Logo: base64 JSON upload → PIL re-encode (`_process_logo`: format+dims header-checked pre-decode, exif_transpose, ≤200px WebP q85, ≤128KB) stored as bytea in the users schema (inside the Postgres backups); served with ETag/304 + nosniff; upload/removal audited with the actor (`guild_logo_uploaded`/`guild_logo_removed` — the abuse trail). `GET /api/recruiting` lists recruiting guilds + member counts via `census_store.latest_guild_member_counts` (guild_history). Daily sweep `backend/server/recruitment_sweep.py` (app lifespan): census lookup BY ID per listed guild — renamed → stored name refreshed, gone → auto-delisted (`recruiting=0`, never deleted), census error → untouched. Erasure tombstones `updated_by`/`logo_uploaded_by`; the logo blob stays (guild asset). Frontend: guild-page Recruitment tab (`pages/guild/GuildRecruitmentTab.tsx`, view for everyone + officer edit) and browse page `pages/RecruitingPage.tsx` at `/recruiting` (nav in BOTH App.tsx BROWSE_ITEMS and MobileNav GROUPS). | +| `backend/server/api/raid_schedule.py` | Guild raid-schedule API. Public `GET /api/guild/{name}/raid-schedule`; officer-or-admin `PUT` (full replace, ≤4 teams × ≤4 raids, each ≤5h, IANA tz, Twitch validated, free text blocklist-screened). Clearing == PUT with empty `teams`. `GET /api/raiding-live` returns the current world's live teams. Tables `raid_teams`/`raid_slots` in the users schema (`backend/server/db/raid_schedule.py`; `raid_slots.days` is `integer[]`). | | `backend/server/raid_live.py` | Twitch-verified "Raiding live" poller (`poll_loop`, `app.py` lifespan). Finds teams inside a scheduled window (team tz, ±15min grace) then verifies live via Twitch Helix; caches per world for `/api/raiding-live`. No-ops without `TWITCH_CLIENT_ID/SECRET`. | | `backend/server/core/twitch.py` | `parse_twitch_login` (accept only `twitch.tv/`) + `is_blocked` (thin wrapper over `text_moderation`). | | `backend/server/core/text_moderation.py` | Shared profanity screen + input sanitiser for officer free text (raid team names/labels) and Twitch logins. `contains_blocked_term` normalises (NFKC, strip invisible/bidi/control chars) then screens via the `better-profanity` package (maintained wordlist + leetspeak variants — no slur list committed in-repo); word-based, so no substring false positives. `sanitize_text` cleans + caps length. Hit → reject + `audit_log`. | @@ -97,7 +97,7 @@ A single deployment serves multiple EQ2 servers, each on its own subdomain (e.g. - **Middleware**: `backend/server/server_context.py` adds `ServerContextMiddleware` which reads the request `Host` header, resolves it to the matching row in the `servers` registry table, and stores it on a contextvar for the lifetime of the request. Non-prod environments also accept a `X-Server` header or `?server=` query-param override for testing. - **Accessors**: all route code calls `current_world()` / `current_server()` (from `backend/server/server_context.py`) rather than the old fixed `WORLD` constant. The bot still uses `WORLD` directly (single-server). -- **Registry**: the `servers` table in `users.db` maps subdomain → world name + per-server settings (`max_level`, `current_xpac`, `launch_dt`, `display_name`). The registry is loaded into memory at startup via `load_registry()` and re-read after admin edits. +- **Registry**: the `servers` table in the users schema maps subdomain → world name + per-server settings (`max_level`, `current_xpac`, `launch_dt`, `display_name`). The registry is loaded into memory at startup via `load_registry()` and re-read after admin edits. - **Per-server data**: character claims, item-watch rows, and parses each carry a `world` column so records are scoped to the server they belong to. Parses are additionally attributed by the `logger_server` field sent by the ACT plugin. - **Universal data**: users, roles, and officer approvals are shared across servers (Discord identity is not server-specific). - **Frontend bootstrap**: `GET /api/server` returns the active server's settings plus a `servers` array for the subdomain switcher; the frontend reads this once on load. @@ -209,19 +209,15 @@ Shared types + className constants for a split page go in a sibling `types.ts`. | `SERVER_MAX_LEVEL` | Seed-only — runtime value is per-server in the `servers` table (admin-editable). Seeds the max character level for the Varsoon row on first migration. | | `LAUNCH_DT` | Seed-only — runtime value is per-server in the `servers` table (admin-editable). Seeds the server launch datetime for the Varsoon row on first migration. | | `ADMIN_DISCORD_IDS` | Comma-separated Discord IDs allowed to hit `/api/admin/*` and delete arbitrary parses | +| `DATABASE_URL` | Postgres DSN for the writable families (users/parses/census/zones/raids — one Supabase database, five schemas). Resolution chain: `DATABASE_URL` → `SUPABASE_DB_URL` → `POSTGRES_CONNECTION_STRING` (`backend/pg.py`). Production = the Supabase SESSION pooler (`:5432`, `sslmode=require`); never the transaction pooler (`:6543`). Migrations in `db/migrations/` apply at app startup. | +| `TEST_DATABASE_URL` | Local PostgreSQL 17 the pytest suite provisions and runs against (default `postgresql://postgres:postgres@localhost:5432/eq2lexicon_test`; created automatically). See `tests/fixtures/pg.py`. | | `PARSE_RETENTION_DAYS` | Days after the fight before the retention sweep (`backend/server/parses/cleanup.py`) hard-deletes a trash parse and collapses a named fight's duplicate uploads to its primary. Default `3`. | +| `PARSE_DETAIL_RETENTION_RAID_DAYS` / `_DUNGEON_DAYS` / `_OTHER_DAYS` | Tiered breakdown-detail retention (defaults 30/14/7): days before a kept fight's `attack_types`+`damage_types` are dropped, by zone category (curated raid / curated group-instance / other). Summary rows live forever. | | `INGEST_CLIENT_LIMITS` | Per-client-app flood protection on `/api/parses/ingest` (`backend/server/core/client_throttle.py`): `=` pairs, `;`-separated, matched case-insensitively as a prefix, bucketed per (app, uploader). Default `EQ2AdvancedDesktop=30/hour` — the third-party client that retried a 500 every ~2 s for nine hours on 2026-09-27. Unlisted clients unaffected; empty disables. | | `TWITCH_CLIENT_ID` / `TWITCH_CLIENT_SECRET` | Twitch app (client-credentials) creds for the raid-schedule "Raiding live" list (`backend/server/raid_live.py`). Both optional — unset ⇒ live list disabled, raid schedule unaffected. | -| `DB_USERS_PATH` | Override the default `data/users.db` location (set on Railway to the persistent-volume mount) | -| `DB_PARSES_PATH` | Override the default `data/backend/server/parses/backend.server.parses.db` location (set on Railway to the persistent-volume mount) | -| `DB_CENSUS_PATH` | Override the default `data/backend/census/census.db` location (persistent last-known character/guild lookups for resilient caching). Set on Railway to the persistent-volume mount; the `.db` is gitignored + generated at runtime. | -| `DB_ZONES_PATH` | Override the default `data/zones/zones.db` location. Set on Railway to the persistent-volume mount (the `.db` itself is not committed — uploaded manually; see "Manual upload: zones.db" below). | -| `DB_RAIDS_PATH` / `DB_ITEMS_PATH` / `DB_SPELLS_PATH` / `DB_RECIPES_PATH` | Same pattern: env-var override of the default `data//.db` location. See `.env.example` for the grouped block. | +| `DB_ITEMS_PATH` / `DB_SPELLS_PATH` / `DB_RECIPES_PATH` | Env-var override of the default `data//.db` location for the remaining READ-ONLY SQLite mirrors (hand-copied to the Railway volume). The former `DB_USERS/PARSES/CENSUS/ZONES/RAIDS_PATH` vars are retired — those families live in Postgres (`DATABASE_URL`). | | `DB_CLASSES_PATH` | **Ignored** (since 2026-09-22, with a startup warning) — classes.db is committed reference data and the repo copy is always authoritative. A stale volume copy behind this override once served the pre-fix swapped Coercer/Illusionist ids in prod. | -| `R2_ENDPOINT` | Litestream backups → `https://.r2.cloudflarestorage.com` | -| `R2_BUCKET` | Litestream backups → bucket name (e.g. `eq2lexicon-backups`) | -| `R2_ACCESS_KEY_ID` | Litestream backups → R2 API token Access Key ID | -| `R2_SECRET_ACCESS_KEY` | Litestream backups → R2 API token Secret Access Key | +| `R2_ENDPOINT` / `R2_BUCKET` / `R2_ACCESS_KEY_ID` / `R2_SECRET_ACCESS_KEY` | Cloudflare R2 bucket for the nightly `pg_dump` backups — set as GITHUB ACTIONS secrets (`.github/workflows/pg-backup.yml`), not Railway vars (litestream is retired). | ## Census API patterns @@ -277,57 +273,40 @@ Two-stage by design — the network-dependent scrape is decoupled from the fast - New slash commands may take up to 1 hour to propagate globally, but appear instantly in the registered guild IDs above - Do not push until the user confirms local testing passes -### Backups (backend.server.parses.db + users.db → Cloudflare R2) +### Backups (Postgres → Cloudflare R2, nightly pg_dump) -[litestream](https://litestream.io) replicates both SQLite DBs to R2 in near-real-time. On container start, `litestream restore -if-replica-exists` rehydrates either DB from R2 if the Railway volume is wiped. Config in `litestream.yml`; orchestration in `railway.toml`'s `startCommand`. - -**One-time R2 setup:** - -1. Sign in at https://dash.cloudflare.com → **R2** → Create bucket (e.g. `eq2lexicon-backups`). -2. Note the **Account ID**; the S3 endpoint is `https://.r2.cloudflarestorage.com`. -3. **R2 → API tab → Create API Token** (NOT "Account API tokens" — that mints Cloudflare REST tokens, not R2 S3 tokens). Permissions: Object Read & Write; specify the bucket. -4. Copy **Access Key ID** and **Secret Access Key** from the success screen. -5. In **Railway → Variables**, add `R2_ENDPOINT`, `R2_BUCKET`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`. -6. Verify in R2 console — `backend/server/parses/` and `users/` prefixes should appear within a minute of next deploy. - -If logs show `cannot fetch generations: Unauthorized` 401s, the wrong token type was minted — re-mint via the R2 API tab specifically. - -**Manual restore:** +The writable families live in Supabase Postgres; `.github/workflows/pg-backup.yml` runs a nightly `pg_dump -Fc` of the five family schemas into the R2 bucket (`pgdump/` prefix) with 30-day in-action pruning. Secrets live on the GitHub repo: `SUPABASE_DB_URL` (session pooler) + the four `R2_*` values. Restore drill: ```bash -litestream restore -config /app/litestream.yml /app/data/backend/server/parses/backend.server.parses.db -litestream restore -config /app/litestream.yml -timestamp 2026-05-24T18:00:00Z -o ./backend.server.parses.db-snapshot /app/data/backend/server/parses/backend.server.parses.db +pg_restore -d "$DATABASE_URL" --clean --if-exists eq2lexicon-.dump ``` -If R2 env vars are absent, the startCommand's `|| true` makes the restore a no-op and the app still runs. +(The litestream/SQLite replication era is over; its R2 prefixes can be deleted ~2 weeks after a healthy cutover. The one-time R2 token how-to: dash.cloudflare.com → R2 → API tab → Create API Token — NOT "Account API tokens" — Object Read & Write on the bucket.) -### Manual upload: zones.db +### Zone metadata refresh (zones schema) -`data/zones/zones.db` is zone-metadata reference data built from the cleaned wiki dump. It's gitignored, lives on the Railway persistent volume, and is replicated by litestream. Boss rosters are **web-editable** — `build_zones_db.py` only writes zones/types/aliases; boss data is curator-managed in-place. - -Refresh procedure (zone metadata only): +Zone metadata is built from the cleaned wiki dump straight into Postgres — no file upload. Boss rosters are **web-editable** curator data; a rebuild upserts `zones` by stable id and fully replaces `zone_types`/`zone_aliases`, never touching `zone_encounters` / `zone_encounter_mobs` (removed zone names ARE pruned, cascading their rosters — the script prints the count; a renamed source zone reads as new+pruned, so check that output). ```powershell python scripts/dev/clean_eq2_zones.py # source → cleaned JSON -python scripts/build_zones_db.py # cleaned JSON → SQLite -python scripts/dev/_smoke_test_zones.py -python scripts/dev/_smoke_test_zones_db.py +uv run python scripts/build_zones_db.py # cleaned JSON → the zones schema (DATABASE_URL) +uv run python scripts/dev/_smoke_test_zones_db.py ``` -Then upload to the Railway volume and set `DB_ZONES_PATH` if your mount differs from `/app/data/zones/zones.db`. Schema changes require a full rebuild + re-upload — rebuilds do **not** touch `zone_encounters` / `zone_encounter_mobs`, so curator boss data is preserved. - -### Manual upload: raids.db (first-time seed) +### Raid strategies seed (raids schema) -`data/raids/raids.db` is hybrid — wiki-seeded strategies plus user edits + revision history. Don't clobber it on every deploy. +Raid strategies are hybrid — wiki-seeded plus user edits + revision history. 1. **First-time seed** (once, before any production user edits): + ```powershell python scripts/dev/scrape_eq2i_raids.py --all-raids - python scripts/dev/ingest_raids_json.py --in scripts/dev/eq2_raid_data.json + uv run python scripts/dev/ingest_raids_json.py --in scripts/dev/eq2_raid_data.json ``` - Upload `data/raids/raids.db` to the Railway volume. Set `DB_RAIDS_PATH` if needed. -2. **Refreshing scraped content**: re-run the scrape, then run ingest in place. `upsert_raid_encounter` skips `SOURCE_MANUAL` rows — human edits survive every re-scrape. + Writes into the raids schema at `DATABASE_URL`. + +2. **Refreshing scraped content**: re-run the scrape, then the ingest. `upsert_raid_encounter` skips `SOURCE_MANUAL` rows — human edits survive every re-scrape. ## Logging conventions diff --git a/tests/conftest.py b/tests/conftest.py index 30cff2f4..ce50fab2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -222,6 +222,7 @@ def mock_character_cache(): from tests.fixtures.logging_state import _logging_state_isolation # noqa: F401,E402 from tests.fixtures.parses_db import parses_db_conn, parses_db_path # noqa: F401,E402 from tests.fixtures.pg import users_schema # noqa: F401,E402 +from tests.fixtures.zones_raids_db import raids_schema, zones_schema # noqa: F401,E402 @pytest.fixture(autouse=True) diff --git a/tests/fixtures/zones_raids_db.py b/tests/fixtures/zones_raids_db.py new file mode 100644 index 00000000..72c78602 --- /dev/null +++ b/tests/fixtures/zones_raids_db.py @@ -0,0 +1,44 @@ +"""Shared zones + raids family fixtures (Postgres schemas). + +``zones_schema`` / ``raids_schema`` lease isolated scratch schemas built +from db/migrations/0004_zones.sql / 0005_raids.sql, re-point the shared +catalogue instances, and yield the schema name — the analog of the old +``XCatalogue(tmp_db)`` + ``catalogue.path`` re-point. +""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest + +from backend.eq2db import raids as raids_module +from backend.eq2db import zones as zones_module +from tests.fixtures.pg import SchemaLeaser + +zones_leaser = SchemaLeaser("zones", "0004_zones.sql") +raids_leaser = SchemaLeaser("raids", "0005_raids.sql") + + +@pytest.fixture +def zones_schema(monkeypatch: pytest.MonkeyPatch) -> Generator[str]: + """Isolated leased zones schema with the shared catalogue re-pointed.""" + name = zones_leaser.acquire() + monkeypatch.setattr(zones_module, "SCHEMA", name) + monkeypatch.setattr(zones_module.catalogue, "schema", name) + try: + yield name + finally: + zones_leaser.release(name) + + +@pytest.fixture +def raids_schema(monkeypatch: pytest.MonkeyPatch) -> Generator[str]: + """Isolated leased raids schema with the shared catalogue re-pointed.""" + name = raids_leaser.acquire() + monkeypatch.setattr(raids_module, "SCHEMA", name) + monkeypatch.setattr(raids_module.catalogue, "schema", name) + try: + yield name + finally: + raids_leaser.release(name) From b90851caaf575d6b6ae046baa3245f65a13c6ba9 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sun, 4 Oct 2026 21:47:35 +0100 Subject: [PATCH 26/32] =?UTF-8?q?test(pg):=20zones=20+=20raids=20test=20su?= =?UTF-8?q?ites=20on=20leased=20schemas=20=E2=80=94=20ALL=20FIVE=20FAMILIE?= =?UTF-8?q?S=20GREEN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twelve files converted by two agents + verified: catalogue-base tests re-based onto a still-SQLite catalogue (RecipeCatalogue) so the SQLite bases stay covered for items/spells/recipes; retired one-time SQLite fixups (zones name normalisation, ACT editor-parity backfill) become descendants pinning their surviving invariants (init_db never mutates rows; legacy alias resolution); "missing DB file" soft-degrades become the documented UndefinedTable contract; explicit identity ids use OVERRIDING SYSTEM VALUE. Full suite: 2153 passed, 0 failed on local PostgreSQL 17 — users, parses, census, zones and raids all on Postgres. ruff + pyright clean, app lifespan boot green, frontend tsc clean. Co-Authored-By: Claude Fable 5 --- tests/eq2db/test_catalogue_base.py | 77 +-- tests/eq2db/test_raids.py | 65 ++- tests/eq2db/test_raids_act.py | 303 ++++++------ .../test_raids_upsert_preserves_overview.py | 38 +- tests/eq2db/test_zones_editable.py | 456 ++++++++---------- tests/server/test_act_categories.py | 21 +- tests/server/test_act_pack.py | 19 +- tests/server/test_act_triggers.py | 4 +- tests/server/test_raid_bosses.py | 86 ++-- tests/server/test_raid_curation.py | 136 +++--- tests/server/test_zone_overview_revisions.py | 34 +- tests/server/test_zones_admin_types.py | 15 +- 12 files changed, 563 insertions(+), 691 deletions(-) diff --git a/tests/eq2db/test_catalogue_base.py b/tests/eq2db/test_catalogue_base.py index 52d05a3c..c3d99778 100644 --- a/tests/eq2db/test_catalogue_base.py +++ b/tests/eq2db/test_catalogue_base.py @@ -1,8 +1,11 @@ -"""Tests for the BaseCatalogue dunder surface (backend/eq2db/_catalogue.py). - -Exercised through RaidCatalogue (no caches, FOREIGN_KEYS=True) and -SpellCatalogue (crc cache) so the behaviour is proven on real subclasses, -not a synthetic stub. +"""Tests for the SQLite base-class dunder surface (backend/db_catalogue.py: +PathBound / BaseCatalogue — still the base for items/spells/recipes/aas/classes). + +Exercised through RecipeCatalogue (no caches) and SpellCatalogue (crc cache) +so the behaviour is proven on real subclasses, not a synthetic stub. +(RaidCatalogue/ZoneCatalogue were the original concrete examples but moved +to PgCatalogue in the Postgres cutover — same intent, SQLite-resident +catalogues now carry the cases.) """ from __future__ import annotations @@ -14,7 +17,7 @@ import pytest from backend.db_catalogue import BaseCatalogue -from backend.eq2db.raids import RaidCatalogue +from backend.eq2db.recipes import RecipeCatalogue from backend.eq2db.spells import SpellCatalogue # --------------------------------------------------------------------------- @@ -24,15 +27,15 @@ class TestRepr: def test_missing_db(self, tmp_path: Path): - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") r = repr(cat) - assert r.startswith("RaidCatalogue(") + assert r.startswith("RecipeCatalogue(") assert "missing" in r # !r-escaped on Windows, so compare against the repr of the string - assert repr(str(tmp_path / "raids.db")) in r + assert repr(str(tmp_path / "recipes.db")) in r def test_ready_db(self, tmp_path: Path): - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") cat.init_db().close() assert "ready" in repr(cat) @@ -43,7 +46,7 @@ def test_cache_sizes_rendered(self, tmp_path: Path): assert "crc_cache=1" in repr(cat) def test_no_cache_subclass_has_no_cache_suffix(self, tmp_path: Path): - r = repr(RaidCatalogue(tmp_path / "raids.db")) + r = repr(RecipeCatalogue(tmp_path / "recipes.db")) assert r.endswith("missing)") @@ -54,10 +57,10 @@ def test_no_cache_subclass_has_no_cache_suffix(self, tmp_path: Path): class TestBool: def test_false_when_db_missing(self, tmp_path: Path): - assert not RaidCatalogue(tmp_path / "nope.db") + assert not RecipeCatalogue(tmp_path / "nope.db") def test_true_when_db_exists(self, tmp_path: Path): - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") cat.init_db().close() assert cat @@ -69,24 +72,24 @@ def test_true_when_db_exists(self, tmp_path: Path): class TestEqHash: def test_same_class_same_path_equal(self, tmp_path: Path): - p = tmp_path / "raids.db" - assert RaidCatalogue(p) == RaidCatalogue(p) - assert hash(RaidCatalogue(p)) == hash(RaidCatalogue(p)) + p = tmp_path / "recipes.db" + assert RecipeCatalogue(p) == RecipeCatalogue(p) + assert hash(RecipeCatalogue(p)) == hash(RecipeCatalogue(p)) def test_different_path_not_equal(self, tmp_path: Path): - assert RaidCatalogue(tmp_path / "a.db") != RaidCatalogue(tmp_path / "b.db") + assert RecipeCatalogue(tmp_path / "a.db") != RecipeCatalogue(tmp_path / "b.db") def test_different_class_same_path_not_equal(self, tmp_path: Path): p = tmp_path / "x.db" - assert RaidCatalogue(p) != SpellCatalogue(p) + assert RecipeCatalogue(p) != SpellCatalogue(p) def test_non_catalogue_not_equal(self, tmp_path: Path): - assert RaidCatalogue(tmp_path / "x.db") != "x.db" + assert RecipeCatalogue(tmp_path / "x.db") != "x.db" def test_usable_as_dict_key(self, tmp_path: Path): - p = tmp_path / "raids.db" - d = {RaidCatalogue(p): "hit"} - assert d[RaidCatalogue(p)] == "hit" + p = tmp_path / "recipes.db" + d = {RecipeCatalogue(p): "hit"} + assert d[RecipeCatalogue(p)] == "hit" # --------------------------------------------------------------------------- @@ -96,18 +99,18 @@ def test_usable_as_dict_key(self, tmp_path: Path): class TestFspath: def test_os_fspath(self, tmp_path: Path): - p = tmp_path / "raids.db" - assert os.fspath(RaidCatalogue(p)) == str(p) + p = tmp_path / "recipes.db" + assert os.fspath(RecipeCatalogue(p)) == str(p) def test_path_conversion(self, tmp_path: Path): - p = tmp_path / "raids.db" - assert Path(RaidCatalogue(p)) == p + p = tmp_path / "recipes.db" + assert Path(RecipeCatalogue(p)) == p def test_sqlite_connect_accepts_catalogue(self, tmp_path: Path): - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") cat.init_db().close() with sqlite3.connect(cat) as conn: - n = conn.execute("SELECT COUNT(*) FROM raid_zones").fetchone()[0] + n = conn.execute("SELECT COUNT(*) FROM recipes").fetchone()[0] conn.close() assert n == 0 @@ -119,17 +122,17 @@ def test_sqlite_connect_accepts_catalogue(self, tmp_path: Path): class TestContextManager: def test_with_yields_initialised_connection(self, tmp_path: Path): - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") with cat as conn: # Schema exists — init_db ran. - n = conn.execute("SELECT COUNT(*) FROM raid_encounters").fetchone()[0] + n = conn.execute("SELECT COUNT(*) FROM recipes").fetchone()[0] assert n == 0 # Connection is CLOSED on exit (unlike sqlite3's own CM). with pytest.raises(sqlite3.ProgrammingError): conn.execute("SELECT 1") def test_nested_with_blocks_close_their_own_conn(self, tmp_path: Path): - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") with cat as outer: with cat as inner: assert inner is not outer @@ -140,7 +143,7 @@ def test_nested_with_blocks_close_their_own_conn(self, tmp_path: Path): outer.execute("SELECT 1") def test_close_happens_on_exception(self, tmp_path: Path): - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") with pytest.raises(RuntimeError): with cat as conn: raise RuntimeError("boom") @@ -158,13 +161,13 @@ def test_unbuilt_schema_degrades_to_empty(self, tmp_path: Path): """File exists but tables don't (fresh volume / stub) -> [] not raise.""" db = tmp_path / "stub.db" sqlite3.connect(db).close() # zero-byte real file, no schema - cat = RaidCatalogue(db) - assert cat._fetchall("SELECT * FROM raid_zones") == [] - assert cat._fetchone("SELECT * FROM raid_zones") is None + cat = RecipeCatalogue(db) + assert cat._fetchall("SELECT * FROM recipes") == [] + assert cat._fetchone("SELECT * FROM recipes") is None def test_other_operational_errors_propagate(self, tmp_path: Path): """Non-schema faults (here: SQL syntax) must NOT be swallowed as empty.""" - cat = RaidCatalogue(tmp_path / "raids.db") + cat = RecipeCatalogue(tmp_path / "recipes.db") cat.init_db().close() with pytest.raises(sqlite3.OperationalError): cat._fetchall("SELEKT broken") @@ -187,7 +190,7 @@ class Broken(BaseCatalogue): # noqa: F841 — definition itself must raise def test_subclass_of_concrete_catalogue_inherits_schema(self, tmp_path: Path): # A test double subclassing a real catalogue is fine — it inherits # the parent's _create_schema. - class Doubled(RaidCatalogue): + class Doubled(RecipeCatalogue): pass Doubled(tmp_path / "x.db").init_db().close() diff --git a/tests/eq2db/test_raids.py b/tests/eq2db/test_raids.py index 325888f7..1bcb53ea 100644 --- a/tests/eq2db/test_raids.py +++ b/tests/eq2db/test_raids.py @@ -1,37 +1,30 @@ -"""Tests for the manual-edit protection in census/raids_db helpers. +"""Tests for the manual-edit protection in the raids catalogue helpers. The encounter helper (upsert_raid_encounter) has always had this protection; the zone helper (upsert_raid_zone) gained it alongside the wiki-seed ingest pipeline so a re-scrape can't overwrite admin/officer-edited zone overviews. + +Postgres edition: each test leases an isolated scratch raids schema via the +``raids_schema`` fixture and opens a schema-scoped pooled connection with +``RaidCatalogue(raids_schema).init_db()`` (the analog of the old +``RaidCatalogue(tmp_path / 'raids.db').init_db()``). Rows are dicts. """ from __future__ import annotations -from pathlib import Path - -import pytest - from backend.eq2db.raids import RaidCatalogue from backend.eq2db.raids import catalogue as raids_db - -@pytest.fixture -def db_path(tmp_path: Path) -> Path: - """Per-test DB so writes don't leak. Returns the file path; callers open - + close their own connection via RaidCatalogue(path).init_db().""" - return tmp_path / "raids.db" - - # --------------------------------------------------------------------------- # upsert_raid_zone — SOURCE_MANUAL preserves markdown on re-scrape # --------------------------------------------------------------------------- -def test_rescrape_preserves_manual_zone_markdown(db_path: Path): +def test_rescrape_preserves_manual_zone_markdown(raids_schema: str): """A human-edited zone (overview/background/access) survives a later SOURCE_SCRAPE upsert intact. Wiki-owned fields (expansion, wiki_url, level_range, last_synced_at) do get refreshed.""" - conn = RaidCatalogue(db_path).init_db() + conn = RaidCatalogue(raids_schema).init_db() try: # First write: simulate a route-level manual edit. raids_db.upsert_raid_zone( @@ -58,27 +51,27 @@ def test_rescrape_preserves_manual_zone_markdown(db_path: Path): row = conn.execute( "SELECT source, overview_md, background_md, access_md, " " wiki_url, level_range, last_synced_at " - "FROM raid_zones WHERE zone_name = ?", + "FROM raid_zones WHERE zone_name = %s", ("The Emerald Halls",), ).fetchone() finally: conn.close() # Manual edits survive untouched. - assert row[0] == "manual" - assert row[1] == "## Our composition\n\n- 2 tanks, 6 healers, 16 dps" - assert row[2] == "Our notes" - assert row[3] == "Our access" + assert row["source"] == "manual" + assert row["overview_md"] == "## Our composition\n\n- 2 tanks, 6 healers, 16 dps" + assert row["background_md"] == "Our notes" + assert row["access_md"] == "Our access" # Wiki-owned fields DID get refreshed (so wiki_url/level_range stay current). - assert row[4] == "https://eq2.fandom.com/wiki/The_Emerald_Halls" - assert row[5] == "80" - assert row[6] is not None # last_synced_at stamped + assert row["wiki_url"] == "https://eq2.fandom.com/wiki/The_Emerald_Halls" + assert row["level_range"] == "80" + assert row["last_synced_at"] is not None # last_synced_at stamped -def test_rescrape_refreshes_when_existing_is_also_scrape(db_path: Path): +def test_rescrape_refreshes_when_existing_is_also_scrape(raids_schema: str): """When the existing row is itself a scrape, a fresh scrape overwrites its markdown — that's how wiki edits propagate.""" - conn = RaidCatalogue(db_path).init_db() + conn = RaidCatalogue(raids_schema).init_db() try: raids_db.upsert_raid_zone( conn, @@ -95,18 +88,18 @@ def test_rescrape_refreshes_when_existing_is_also_scrape(db_path: Path): source=raids_db.SOURCE_SCRAPE, ) row = conn.execute( - "SELECT overview_md FROM raid_zones WHERE zone_name = ?", + "SELECT overview_md FROM raid_zones WHERE zone_name = %s", ("The Emerald Halls",), ).fetchone() finally: conn.close() - assert row[0] == "NEW WIKI" + assert row["overview_md"] == "NEW WIKI" -def test_manual_upsert_creates_new_row(db_path: Path): +def test_manual_upsert_creates_new_row(raids_schema: str): """A first-write with SOURCE_MANUAL inserts cleanly (no existing row to protect). Used by the lazy-create path in _write_overview_sync.""" - conn = RaidCatalogue(db_path).init_db() + conn = RaidCatalogue(raids_schema).init_db() try: raids_db.upsert_raid_zone( conn, @@ -116,12 +109,12 @@ def test_manual_upsert_creates_new_row(db_path: Path): source=raids_db.SOURCE_MANUAL, ) row = conn.execute( - "SELECT source, overview_md FROM raid_zones WHERE zone_name = ?", + "SELECT source, overview_md FROM raid_zones WHERE zone_name = %s", ("Veeshan's Peak",), ).fetchone() finally: conn.close() - assert row == ("manual", "hello") + assert (row["source"], row["overview_md"]) == ("manual", "hello") # --------------------------------------------------------------------------- @@ -130,8 +123,8 @@ def test_manual_upsert_creates_new_row(db_path: Path): # --------------------------------------------------------------------------- -def test_rescrape_preserves_manual_encounter_strategy(db_path: Path): - conn = RaidCatalogue(db_path).init_db() +def test_rescrape_preserves_manual_encounter_strategy(raids_schema: str): + conn = RaidCatalogue(raids_schema).init_db() try: zone_id = raids_db.upsert_raid_zone( conn, @@ -157,10 +150,10 @@ def test_rescrape_preserves_manual_encounter_strategy(db_path: Path): source=raids_db.SOURCE_SCRAPE, ) row = conn.execute( - "SELECT source, strategy_md FROM raid_encounters WHERE raid_zone_id = ? AND mob_name_lower = ?", + "SELECT source, strategy_md FROM raid_encounters WHERE raid_zone_id = %s AND mob_name_lower = %s", (zone_id, "prince thirneg"), ).fetchone() finally: conn.close() - assert row[0] == "manual" - assert row[1] == "OUR STRAT" + assert row["source"] == "manual" + assert row["strategy_md"] == "OUR STRAT" diff --git a/tests/eq2db/test_raids_act.py b/tests/eq2db/test_raids_act.py index 30b785e9..d819f0c8 100644 --- a/tests/eq2db/test_raids_act.py +++ b/tests/eq2db/test_raids_act.py @@ -1,25 +1,24 @@ -"""Tests for census.raids_act_db — COV-010. +"""Tests for the ACT trigger/spell-timer helpers on the raids catalogue — COV-010. -Uses in-memory SQLite per test (via census.raids_db.init_db). Covers: -- list_act_triggers_for_encounter ordering + missing-path fallback +Postgres edition: each test leases an isolated scratch raids schema (the +``raids_schema`` fixture) with one seeded zone + encounter. Covers: +- list_act_triggers_for_encounter ordering + unprovisioned-schema contract - get_act_trigger unknown id → None - upsert_act_trigger INSERT vs UPDATE + edited_by stamp - delete_act_trigger returns True/False - Spell-timer helpers (same shape) - upsert_act_spell_timer name_lower UNIQUE collision - -Target: ≥ 80% on census.raids_act_db. +- the migrations-owned enrichment DDL (descendants of the SQLite + migration/backfill tests) """ from __future__ import annotations -import sqlite3 -import tempfile -from pathlib import Path - +import psycopg.errors import pytest from backend.eq2db.raids import RaidCatalogue +from tests.fixtures.pg import pg_conn # Conn-taking write helpers are staticmethods — alias for readable call sites. upsert_act_trigger = RaidCatalogue.upsert_act_trigger @@ -29,59 +28,62 @@ @pytest.fixture -def db_path(tmp_path: Path) -> Path: - """A fresh raids.db (file-backed via tmp_path) with one zone + encounter.""" - p = tmp_path / "raids.db" - conn = RaidCatalogue(p).init_db() - # Insert a seed raid_zone + encounter so FK-style references are valid - conn.execute( - "INSERT INTO raid_zones (zone_name, zone_name_lower, expansion_short, source) " - "VALUES ('Test Zone', 'test zone', 'TS', 'manual')" - ) - zone_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0] - conn.execute( - "INSERT INTO raid_encounters (raid_zone_id, mob_name, mob_name_lower, source) " - "VALUES (?, 'Boss One', 'boss one', 'manual')", - (zone_id,), - ) - conn.commit() - conn.close() - return p +def db(raids_schema: str) -> str: + """A fresh scratch raids schema with one zone + encounter seeded — + the analog of the old tmp_path raids.db fixture. Returns the schema name.""" + with pg_conn(raids_schema) as conn: + zone_id = conn.execute( + "INSERT INTO raid_zones (zone_name, zone_name_lower, expansion_short, source) " + "VALUES ('Test Zone', 'test zone', 'TS', 'manual') RETURNING id" + ).fetchone()["id"] + conn.execute( + "INSERT INTO raid_encounters (raid_zone_id, mob_name, mob_name_lower, source) " + "VALUES (%s, 'Boss One', 'boss one', 'manual')", + (zone_id,), + ) + return raids_schema @pytest.fixture -def enc_id(db_path: Path) -> int: - with sqlite3.connect(db_path) as conn: +def enc_id(db: str) -> int: + with pg_conn(db) as conn: row = conn.execute("SELECT id FROM raid_encounters LIMIT 1").fetchone() - return row[0] + return row["id"] @pytest.fixture -def db_conn(db_path: Path): - conn = sqlite3.connect(db_path) +def db_conn(db: str): + conn = RaidCatalogue(db).init_db() yield conn conn.close() +#: An unprovisioned schema name — the Pg analog of the old "missing path". +#: SchemaBound documents no exists-degrade: migrations run before the app +#: serves, so a missing relation is a real fault (raises), never a soft +#: empty result the way the SQLite catalogues degraded. +_MISSING = "raids_pytest_never_provisioned" + + # --------------------------------------------------------------------------- # ACT Trigger helpers # --------------------------------------------------------------------------- class TestListActTriggersForEncounter: - def test_returns_empty_when_path_missing(self, tmp_path: Path): - missing = tmp_path / "nonexistent.db" - assert RaidCatalogue(missing).list_act_triggers_for_encounter(1) == [] + def test_raises_on_unprovisioned_schema(self): + with pytest.raises(psycopg.errors.UndefinedTable): + RaidCatalogue(_MISSING).list_act_triggers_for_encounter(1) - def test_returns_empty_for_unknown_encounter(self, db_path: Path): - assert RaidCatalogue(db_path).list_act_triggers_for_encounter(9999) == [] + def test_returns_empty_for_unknown_encounter(self, db: str): + assert RaidCatalogue(db).list_act_triggers_for_encounter(9999) == [] - def test_ordering_by_position_then_id(self, db_path: Path, db_conn, enc_id: int): + def test_ordering_by_position_then_id(self, db: str, db_conn, enc_id: int): # Insert triggers with shuffled positions upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="c", position=2) upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="a", position=0) upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="b", position=1) - rows = RaidCatalogue(db_path).list_act_triggers_for_encounter(enc_id) + rows = RaidCatalogue(db).list_act_triggers_for_encounter(enc_id) assert len(rows) == 3 assert rows[0]["regex"] == "a" assert rows[1]["regex"] == "b" @@ -89,15 +91,16 @@ def test_ordering_by_position_then_id(self, db_path: Path, db_conn, enc_id: int) class TestGetActTrigger: - def test_returns_none_when_path_missing(self, tmp_path: Path): - assert RaidCatalogue(tmp_path / "no.db").get_act_trigger(1) is None + def test_raises_on_unprovisioned_schema(self): + with pytest.raises(psycopg.errors.UndefinedTable): + RaidCatalogue(_MISSING).get_act_trigger(1) - def test_returns_none_for_unknown_id(self, db_path: Path): - assert RaidCatalogue(db_path).get_act_trigger(9999) is None + def test_returns_none_for_unknown_id(self, db: str): + assert RaidCatalogue(db).get_act_trigger(9999) is None - def test_returns_dict_for_existing_trigger(self, db_path: Path, db_conn, enc_id: int): + def test_returns_dict_for_existing_trigger(self, db: str, db_conn, enc_id: int): trigger_id = upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="test-regex", label="Boss Pull") - row = RaidCatalogue(db_path).get_act_trigger(trigger_id) + row = RaidCatalogue(db).get_act_trigger(trigger_id) assert row is not None assert row["regex"] == "test-regex" assert row["label"] == "Boss Pull" @@ -114,14 +117,14 @@ def test_update_returns_same_id(self, db_conn, enc_id: int): returned = upsert_act_trigger(db_conn, trigger_id=tid, raid_encounter_id=enc_id, regex="updated") assert returned == tid - def test_stamps_edited_by(self, db_path: Path, db_conn, enc_id: int): + def test_stamps_edited_by(self, db: str, db_conn, enc_id: int): tid = upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="x", edited_by="user-123") - row = RaidCatalogue(db_path).get_act_trigger(tid) + row = RaidCatalogue(db).get_act_trigger(tid) assert row["last_edited_by"] == "user-123" - def test_stamps_last_edited_at(self, db_path: Path, db_conn, enc_id: int): + def test_stamps_last_edited_at(self, db: str, db_conn, enc_id: int): tid = upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="x") - row = RaidCatalogue(db_path).get_act_trigger(tid) + row = RaidCatalogue(db).get_act_trigger(tid) assert row["last_edited_at"] is not None and row["last_edited_at"] > 0 @@ -133,10 +136,10 @@ def test_returns_true_when_deleted(self, db_conn, enc_id: int): def test_returns_false_for_unknown_id(self, db_conn): assert delete_act_trigger(db_conn, 9999) is False - def test_row_gone_after_delete(self, db_path: Path, db_conn, enc_id: int): + def test_row_gone_after_delete(self, db: str, db_conn, enc_id: int): tid = upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="gone") delete_act_trigger(db_conn, tid) - assert RaidCatalogue(db_path).get_act_trigger(tid) is None + assert RaidCatalogue(db).get_act_trigger(tid) is None # --------------------------------------------------------------------------- @@ -145,31 +148,33 @@ def test_row_gone_after_delete(self, db_path: Path, db_conn, enc_id: int): class TestListActSpellTimersForEncounter: - def test_returns_empty_when_path_missing(self, tmp_path: Path): - assert RaidCatalogue(tmp_path / "no.db").list_act_spell_timers_for_encounter(1) == [] + def test_raises_on_unprovisioned_schema(self): + with pytest.raises(psycopg.errors.UndefinedTable): + RaidCatalogue(_MISSING).list_act_spell_timers_for_encounter(1) - def test_returns_empty_for_unknown_encounter(self, db_path: Path): - assert RaidCatalogue(db_path).list_act_spell_timers_for_encounter(9999) == [] + def test_returns_empty_for_unknown_encounter(self, db: str): + assert RaidCatalogue(db).list_act_spell_timers_for_encounter(9999) == [] - def test_returns_inserted_timer(self, db_path: Path, db_conn, enc_id: int): + def test_returns_inserted_timer(self, db: str, db_conn, enc_id: int): upsert_act_spell_timer(db_conn, raid_encounter_id=enc_id, name="Deathmark", timer_duration_s=30) - rows = RaidCatalogue(db_path).list_act_spell_timers_for_encounter(enc_id) + rows = RaidCatalogue(db).list_act_spell_timers_for_encounter(enc_id) assert len(rows) == 1 assert rows[0]["name"] == "Deathmark" class TestGetActSpellTimer: - def test_returns_none_when_path_missing(self, tmp_path: Path): - assert RaidCatalogue(tmp_path / "no.db").get_act_spell_timer(1) is None + def test_raises_on_unprovisioned_schema(self): + with pytest.raises(psycopg.errors.UndefinedTable): + RaidCatalogue(_MISSING).get_act_spell_timer(1) - def test_returns_none_for_unknown_id(self, db_path: Path): - assert RaidCatalogue(db_path).get_act_spell_timer(9999) is None + def test_returns_none_for_unknown_id(self, db: str): + assert RaidCatalogue(db).get_act_spell_timer(9999) is None - def test_returns_dict_for_existing_timer(self, db_path: Path, db_conn, enc_id: int): + def test_returns_dict_for_existing_timer(self, db: str, db_conn, enc_id: int): timer_id = upsert_act_spell_timer( db_conn, raid_encounter_id=enc_id, name="Arcane Distortion", timer_duration_s=60 ) - row = RaidCatalogue(db_path).get_act_spell_timer(timer_id) + row = RaidCatalogue(db).get_act_spell_timer(timer_id) assert row is not None assert row["name"] == "Arcane Distortion" assert row["timer_duration_s"] == 60 @@ -188,21 +193,21 @@ def test_update_returns_same_id(self, db_conn, enc_id: int): ) assert returned == tid - def test_stamps_edited_by(self, db_path: Path, db_conn, enc_id: int): + def test_stamps_edited_by(self, db: str, db_conn, enc_id: int): tid = upsert_act_spell_timer( db_conn, raid_encounter_id=enc_id, name="Spell Gamma", timer_duration_s=15, edited_by="officer-1" ) - row = RaidCatalogue(db_path).get_act_spell_timer(tid) + row = RaidCatalogue(db).get_act_spell_timer(tid) assert row["last_edited_by"] == "officer-1" - def test_name_lower_stored_lowercase(self, db_path: Path, db_conn, enc_id: int): + def test_name_lower_stored_lowercase(self, db: str, db_conn, enc_id: int): tid = upsert_act_spell_timer(db_conn, raid_encounter_id=enc_id, name="Camelcase Spell", timer_duration_s=5) - row = RaidCatalogue(db_path).get_act_spell_timer(tid) + row = RaidCatalogue(db).get_act_spell_timer(tid) assert row["name_lower"] == "camelcase spell" def test_unique_collision_raises_integrity_error(self, db_conn, enc_id: int): upsert_act_spell_timer(db_conn, raid_encounter_id=enc_id, name="Unique Spell", timer_duration_s=5) - with pytest.raises(sqlite3.IntegrityError): + with pytest.raises(psycopg.errors.UniqueViolation): upsert_act_spell_timer(db_conn, raid_encounter_id=enc_id, name="Unique Spell", timer_duration_s=10) @@ -221,7 +226,7 @@ def test_returns_false_for_unknown_id(self, db_conn): class TestEnrichment: - def test_spell_timer_enrichment_round_trips(self, db_path: Path, db_conn, enc_id: int): + def test_spell_timer_enrichment_round_trips(self, db: str, db_conn, enc_id: int): tid = upsert_act_spell_timer( db_conn, raid_encounter_id=enc_id, @@ -230,110 +235,74 @@ def test_spell_timer_enrichment_round_trips(self, db_path: Path, db_conn, enc_id damage_type="poison, disease", control_effect="stifle", ) - row = RaidCatalogue(db_path).get_act_spell_timer(tid) + row = RaidCatalogue(db).get_act_spell_timer(tid) assert row is not None assert row["damage_type"] == "poison, disease" assert row["control_effect"] == "stifle" - def test_trigger_cooldown_round_trips(self, db_path: Path, db_conn, enc_id: int): + def test_trigger_cooldown_round_trips(self, db: str, db_conn, enc_id: int): trig_id = upsert_act_trigger(db_conn, raid_encounter_id=enc_id, regex="Feed, my pets!", cooldown_seconds=2.5) - row = RaidCatalogue(db_path).get_act_trigger(trig_id) + row = RaidCatalogue(db).get_act_trigger(trig_id) assert row is not None assert row["cooldown_seconds"] == 2.5 - def test_enrichment_defaults_are_empty(self, db_path: Path, db_conn, enc_id: int): + def test_enrichment_defaults_are_empty(self, db: str, db_conn, enc_id: int): tid = upsert_act_spell_timer(db_conn, raid_encounter_id=enc_id, name="Plain", timer_duration_s=30) - row = RaidCatalogue(db_path).get_act_spell_timer(tid) + row = RaidCatalogue(db).get_act_spell_timer(tid) assert row is not None assert (row["damage_type"], row["control_effect"]) == ("", "") - def test_init_db_migrates_a_pre_enrichment_db(self, tmp_path: Path): - """A raids.db created before the enrichment columns existed must - gain them on init_db (the _apply_migrations ALTERs) — the exact - failure mode the test-migrations-against-old-DB-shape rule exists - for.""" - p = tmp_path / "old-shape.db" - conn = sqlite3.connect(p) - # The pre-2026-08 table shapes, frozen: no cooldown_seconds / - # damage_type / control_effect. - conn.execute( - """ - CREATE TABLE act_triggers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raid_encounter_id INTEGER NOT NULL, - position INTEGER NOT NULL DEFAULT 0, - label TEXT, notes TEXT, - active INTEGER NOT NULL DEFAULT 1, - regex TEXT NOT NULL, - sound_data TEXT NOT NULL DEFAULT '', - sound_type INTEGER NOT NULL DEFAULT 3, - category_restrict INTEGER NOT NULL DEFAULT 0, - category TEXT, - timer INTEGER NOT NULL DEFAULT 0, - timer_name TEXT, - tabbed INTEGER NOT NULL DEFAULT 0, - last_edited_at INTEGER, last_edited_by TEXT, - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) - ) - """ - ) - conn.execute( - """ - CREATE TABLE act_spell_timers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raid_encounter_id INTEGER NOT NULL, - name TEXT NOT NULL, name_lower TEXT NOT NULL, - checked INTEGER NOT NULL DEFAULT 0, - timer_duration_s INTEGER NOT NULL, - only_master_ticks INTEGER NOT NULL DEFAULT 0, - restrict INTEGER NOT NULL DEFAULT 0, - absolute_ INTEGER NOT NULL DEFAULT 0, - start_wav TEXT NOT NULL DEFAULT '', - warning_wav TEXT NOT NULL DEFAULT '', - warning_value INTEGER NOT NULL DEFAULT 10, - radial_display INTEGER NOT NULL DEFAULT 0, - modable INTEGER NOT NULL DEFAULT 0, - tooltip TEXT NOT NULL DEFAULT '', - fill_color INTEGER NOT NULL DEFAULT -16776961, - panel1 INTEGER NOT NULL DEFAULT 1, - panel2 INTEGER NOT NULL DEFAULT 0, - remove_value INTEGER NOT NULL DEFAULT -15, - category TEXT, - restrict_category INTEGER NOT NULL DEFAULT 0, - last_edited_at INTEGER, last_edited_by TEXT, - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - UNIQUE (raid_encounter_id, name_lower) - ) - """ - ) - conn.execute( - "INSERT INTO act_spell_timers (raid_encounter_id, name, name_lower, timer_duration_s) " - "VALUES (1, 'Old Timer', 'old timer', 30)" - ) - conn.commit() - conn.close() - - RaidCatalogue(p).init_db().close() - - with sqlite3.connect(p) as check: - trig_cols = {r[1] for r in check.execute("PRAGMA table_info(act_triggers)")} - timer_cols = {r[1] for r in check.execute("PRAGMA table_info(act_spell_timers)")} + def test_enrichment_columns_in_migrations_owned_ddl(self, db: str, enc_id: int): + """Descendant of the SQLite pre-enrichment migration test (init_db + _apply_migrations ALTERs): on Postgres the enrichment columns are + part of the migrations-owned DDL outright (db/migrations/ + 0005_raids.sql). A raw INSERT that omits them — the shape a + pre-enrichment writer produced — reads back the defaults the old + ALTER backfilled.""" + with pg_conn(db) as conn: + trig_cols = { + r["column_name"] + for r in conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'act_triggers'", + (db,), + ) + } + timer_cols = { + r["column_name"] + for r in conn.execute( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = %s AND table_name = 'act_spell_timers'", + (db,), + ) + } assert "cooldown_seconds" in trig_cols assert {"damage_type", "control_effect"} <= timer_cols - # Pre-existing rows read back with the defaults. - row = check.execute( - "SELECT damage_type, control_effect FROM act_spell_timers WHERE name = 'Old Timer'" + + # Pre-enrichment-shaped writes get the defaults. + timer_row = conn.execute( + "INSERT INTO act_spell_timers (raid_encounter_id, name, name_lower, timer_duration_s) " + "VALUES (%s, 'Old Timer', 'old timer', 30) " + "RETURNING damage_type, control_effect", + (enc_id,), + ).fetchone() + assert (timer_row["damage_type"], timer_row["control_effect"]) == ("", "") + trig_row = conn.execute( + "INSERT INTO act_triggers (raid_encounter_id, regex) VALUES (%s, 'x') RETURNING cooldown_seconds", + (enc_id,), ).fetchone() - assert row == ("", "") - # Idempotent: a second init on the migrated file is a no-op. - RaidCatalogue(p).init_db().close() + assert trig_row["cooldown_seconds"] == 1.0 class TestEditorParityBackfill: - def test_backfill_flips_editor_stripped_fields_once(self, db_path: Path, db_conn, enc_id: int): - """Rows saved by the pre-parity web editor (modable/checked 0, - blank sounds) flip to the real defaults exactly ONCE — a curator's - later deliberate zero survives re-init (meta-guarded).""" + def test_init_db_never_mutates_rows(self, db: str, db_conn, enc_id: int): + """Descendant of the SQLite one-time editor-parity backfill test + (meta-guarded flip of pre-parity 'stripped' rows): the backfill was + retired at the Postgres cutover — its repair crossed over in the + copied data, and ``PgCatalogue.init_db()`` is a pooled-connection + handle that never rewrites rows. The surviving contract is the + second half of the old test's intent: a curator's deliberate + zeros/blanks survive any number of re-inits verbatim.""" tid = upsert_act_spell_timer( db_conn, raid_encounter_id=enc_id, @@ -354,26 +323,24 @@ def test_backfill_flips_editor_stripped_fields_once(self, db_path: Path, db_conn start_wav="", warning_wav="custom.wav", ) - # Clear the guard the fixture's init already set, then re-init. - db_conn.execute("DELETE FROM _meta WHERE key = 'act_editor_parity_backfill'") - db_conn.commit() - RaidCatalogue(db_path).init_db().close() + # Re-init twice — on Pg this opens (and closes) pooled connections + # and nothing else: no normalization, no backfill, no meta guard. + RaidCatalogue(db).init_db().close() + RaidCatalogue(db).init_db().close() - cat = RaidCatalogue(db_path) + cat = RaidCatalogue(db) row = cat.get_act_spell_timer(tid) assert row is not None - assert (row["modable"], row["checked"]) == (1, 1) - assert (row["start_wav"], row["warning_wav"]) == ("tts", "tts") - # One sound set deliberately → the pair is NOT blank-both → kept. + assert (row["modable"], row["checked"]) == (0, 0) + assert (row["start_wav"], row["warning_wav"]) == ("", "") partial = cat.get_act_spell_timer(deliberate) assert partial is not None assert (partial["start_wav"], partial["warning_wav"]) == ("", "custom.wav") - # Post-backfill: a deliberate off survives the next init. - with sqlite3.connect(db_path) as conn: - conn.execute("UPDATE act_spell_timers SET modable = 0 WHERE id = ?", (tid,)) - conn.commit() - RaidCatalogue(db_path).init_db().close() + # A later deliberate zero also survives the next init. + with pg_conn(db) as conn: + conn.execute("UPDATE act_spell_timers SET modable = 0 WHERE id = %s", (tid,)) + RaidCatalogue(db).init_db().close() row = cat.get_act_spell_timer(tid) assert row is not None assert row["modable"] == 0 diff --git a/tests/eq2db/test_raids_upsert_preserves_overview.py b/tests/eq2db/test_raids_upsert_preserves_overview.py index d036f179..69130698 100644 --- a/tests/eq2db/test_raids_upsert_preserves_overview.py +++ b/tests/eq2db/test_raids_upsert_preserves_overview.py @@ -10,6 +10,9 @@ 'don't touch' instead of 'clobber to NULL'. Destructive writes go through targeted UPDATEs (e.g. _update_overview_sync), not through upsert_raid_zone. + +Postgres edition: leased scratch schema via ``raids_schema``; dict rows, +%s params. """ from __future__ import annotations @@ -18,12 +21,11 @@ from backend.eq2db.raids import catalogue as raids_db -def test_upsert_with_none_overview_preserves_existing_overview(tmp_path): +def test_upsert_with_none_overview_preserves_existing_overview(raids_schema: str): """The regression scenario: an existing zone has a curator-written overview_md. A subsequent upsert_raid_zone call WITHOUT overview_md (the strategy-write path) must NOT wipe it to NULL.""" - db = tmp_path / "raids.db" - conn = RaidCatalogue(db).init_db() + conn = RaidCatalogue(raids_schema).init_db() try: # 1. Curator writes an overview — overview_md is set. raids_db.upsert_raid_zone( @@ -34,11 +36,11 @@ def test_upsert_with_none_overview_preserves_existing_overview(tmp_path): source=raids_db.SOURCE_MANUAL, ) row = conn.execute( - "SELECT overview_md, source FROM raid_zones WHERE zone_name = ?", + "SELECT overview_md, source FROM raid_zones WHERE zone_name = %s", ("Mistmoore's Inner Sanctum",), ).fetchone() - assert row[0] == "Bring poison cures. Stagger interrupts on Mob B." - assert row[1] == raids_db.SOURCE_MANUAL + assert row["overview_md"] == "Bring poison cures. Stagger interrupts on Mob B." + assert row["source"] == raids_db.SOURCE_MANUAL # 2. Curator edits a boss strategy — _write_strategy_sync auto-creates # the zone parent by calling upsert_raid_zone() WITHOUT overview_md. @@ -51,20 +53,19 @@ def test_upsert_with_none_overview_preserves_existing_overview(tmp_path): # overview_md not passed — defaults to None ) row = conn.execute( - "SELECT overview_md, source FROM raid_zones WHERE zone_name = ?", + "SELECT overview_md, source FROM raid_zones WHERE zone_name = %s", ("Mistmoore's Inner Sanctum",), ).fetchone() # The fix: overview_md is preserved, not nulled. - assert row[0] == "Bring poison cures. Stagger interrupts on Mob B." - assert row[1] == raids_db.SOURCE_MANUAL + assert row["overview_md"] == "Bring poison cures. Stagger interrupts on Mob B." + assert row["source"] == raids_db.SOURCE_MANUAL finally: conn.close() -def test_upsert_with_none_access_md_preserves_existing(tmp_path): +def test_upsert_with_none_access_md_preserves_existing(raids_schema: str): """Same defensive contract for access_md.""" - db = tmp_path / "raids.db" - conn = RaidCatalogue(db).init_db() + conn = RaidCatalogue(raids_schema).init_db() try: raids_db.upsert_raid_zone( conn, @@ -79,13 +80,13 @@ def test_upsert_with_none_access_md_preserves_existing(tmp_path): expansion_short="EoF", source=raids_db.SOURCE_MANUAL, ) - row = conn.execute("SELECT access_md FROM raid_zones WHERE zone_name = ?", ("X",)).fetchone() - assert row[0] == "Get to the back of the zone via the side passage." + row = conn.execute("SELECT access_md FROM raid_zones WHERE zone_name = %s", ("X",)).fetchone() + assert row["access_md"] == "Get to the back of the zone via the side passage." finally: conn.close() -def test_upsert_with_non_none_overview_overwrites(tmp_path): +def test_upsert_with_non_none_overview_overwrites(raids_schema: str): """The fix must NOT break the case where a caller DOES want to write a fresh overview_md — only None means 'don't touch'. A non-None value still overwrites the existing row. @@ -93,8 +94,7 @@ def test_upsert_with_non_none_overview_overwrites(tmp_path): This is the wiki re-scrape case where source=SCRAPE hits an existing source=SCRAPE row: the markdown should be refreshed with the latest wiki content.""" - db = tmp_path / "raids.db" - conn = RaidCatalogue(db).init_db() + conn = RaidCatalogue(raids_schema).init_db() try: raids_db.upsert_raid_zone( conn, @@ -110,7 +110,7 @@ def test_upsert_with_non_none_overview_overwrites(tmp_path): overview_md="New wiki content", source=raids_db.SOURCE_SCRAPE, ) - row = conn.execute("SELECT overview_md FROM raid_zones WHERE zone_name = ?", ("Y",)).fetchone() - assert row[0] == "New wiki content" + row = conn.execute("SELECT overview_md FROM raid_zones WHERE zone_name = %s", ("Y",)).fetchone() + assert row["overview_md"] == "New wiki content" finally: conn.close() diff --git a/tests/eq2db/test_zones_editable.py b/tests/eq2db/test_zones_editable.py index 199d0c35..73a2944b 100644 --- a/tests/eq2db/test_zones_editable.py +++ b/tests/eq2db/test_zones_editable.py @@ -1,372 +1,324 @@ -"""Tests for the editable raid-roster helpers added to zones_db.""" +"""Tests for the editable raid-roster helpers on the zones catalogue. + +Postgres edition: tests lease an isolated scratch schema via the +``zones_schema`` fixture (tests/fixtures/zones_raids_db.py) and construct +``ZoneCatalogue(zones_schema)`` — the analog of the old +``ZoneCatalogue(tmp_db)``. Raw seeding/assertions go through +``pg_conn(schema)`` (dict rows, %s params). + +Note on the two former init_db data-normalization tests: the one-time +SQLite fixups (comma-joined encounter_name collapse, " (Zone)" suffix +strip) were retired in the Postgres cutover — ``PgCatalogue.init_db()`` +creates and mutates nothing (DDL is owned by db/migrations/; prod data +crossed over already normalized). The descendants below pin that new +contract plus the surviving alias-resolution behaviour. +""" from __future__ import annotations -import sqlite3 +import pytest from backend.eq2db import zones as zones_db +from tests.fixtures.pg import pg_conn -def _seed_legacy_zone(path) -> tuple[int, int]: - """Build a minimal zones.db with one zone + one comma-joined encounter - (two mobs at positions 0 + 1). Returns (zone_id, encounter_id).""" - conn = zones_db.ZoneCatalogue(path).init_db() - try: +def _seed_legacy_zone(schema: str) -> tuple[int, int]: + """Seed one zone + one comma-joined encounter (two mobs at positions + 0 + 1) — the legacy pre-normalization shape. Returns (zone_id, encounter_id).""" + with pg_conn(schema) as conn: conn.execute( - "INSERT INTO zones (name, name_lower, expansion_short, expansion_name, " + "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, " "expansion_confidence, expansion_source) " - "VALUES ('Shard of Hate', 'shard of hate', 'RoK', 'Rise of Kunark', 'test', 'test')" - ) - zone_id = conn.execute("SELECT id FROM zones WHERE name = 'Shard of Hate'").fetchone()[0] - conn.execute( - "INSERT INTO zone_encounters (zone_id, encounter_name, position) VALUES (?, 'Ire, Malevolence', 3)", - (zone_id,), + "VALUES (1, 'Shard of Hate', 'shard of hate', 'RoK', 'Rise of Kunark', 'test', 'test')" ) + zone_id = 1 enc_id = conn.execute( - "SELECT id FROM zone_encounters WHERE zone_id = ? AND position = 3", (zone_id,) - ).fetchone()[0] + "INSERT INTO zone_encounters (zone_id, encounter_name, position) " + "VALUES (%s, 'Ire, Malevolence', 3) RETURNING id", + (zone_id,), + ).fetchone()["id"] conn.execute( "INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) " - "VALUES (?, 'Ire', 'ire', 0)", + "VALUES (%s, 'Ire', 'ire', 0)", (enc_id,), ) conn.execute( "INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) " - "VALUES (?, 'Malevolence', 'malevolence', 1)", + "VALUES (%s, 'Malevolence', 'malevolence', 1)", (enc_id,), ) - conn.commit() return zone_id, enc_id - finally: - conn.close() -def test_init_db_normalizes_comma_joined_encounter_name(tmp_path): - """A legacy encounter whose name is the comma-joined mob list is rewritten - to the position-0 mob's name. Non-comma names are left alone. Idempotent.""" - p = tmp_path / "zones.db" - zone_id, enc_id = _seed_legacy_zone(p) - # Add a non-comma encounter that should NOT be touched. - with sqlite3.connect(p) as conn: - conn.execute( - "INSERT INTO zone_encounters (zone_id, encounter_name, position) VALUES (?, 'Demetrius Crane', 1)", - (zone_id,), - ) +def test_init_db_is_schema_passive_no_data_normalization(zones_schema): + """Descendant of the SQLite comma-joined-encounter-name normalization + test: on Postgres ``init_db()`` is a pooled-connection handle, NOT a + schema/data fixup pass — a legacy-shaped row passes through untouched + (the one-time normalization was retired at cutover; the copied prod + data was already normalized). Pins the "init_db never mutates data" + contract so a future destructive _post_init analog is caught.""" + zone_id, enc_id = _seed_legacy_zone(zones_schema) + # Add a non-comma encounter alongside. + with pg_conn(zones_schema) as conn: enc_id2 = conn.execute( - "SELECT id FROM zone_encounters WHERE zone_id = ? AND position = 1", (zone_id,) - ).fetchone()[0] + "INSERT INTO zone_encounters (zone_id, encounter_name, position) " + "VALUES (%s, 'Demetrius Crane', 1) RETURNING id", + (zone_id,), + ).fetchone()["id"] conn.execute( "INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) " - "VALUES (?, 'Demetrius Crane', 'demetrius crane', 0)", + "VALUES (%s, 'Demetrius Crane', 'demetrius crane', 0)", (enc_id2,), ) - conn.commit() - - # Re-init to trigger normalization (init_db is idempotent). - conn = zones_db.ZoneCatalogue(p).init_db() - try: - rows = {r[0]: r[1] for r in conn.execute("SELECT id, encounter_name FROM zone_encounters")} - assert rows[enc_id] == "Ire" # comma-joined collapsed to primary - assert rows[enc_id2] == "Demetrius Crane" # untouched - finally: - conn.close() - - # Second run is a no-op (encounter_name no longer contains a comma). - conn = zones_db.ZoneCatalogue(p).init_db() - try: - assert conn.execute("SELECT encounter_name FROM zone_encounters WHERE id = ?", (enc_id,)).fetchone()[0] == "Ire" - finally: - conn.close() - - -def test_init_db_strips_zone_suffix_from_zone_names(tmp_path): - """A wiki-import zone name ending in ' (Zone)' is rewritten to the bare - name, and the parenthesised form is preserved as an alias so historic - references still resolve. Non-suffixed names are left alone. Idempotent.""" - p = tmp_path / "zones.db" - conn = zones_db.ZoneCatalogue(p).init_db() - try: + + # init_db is idempotent and side-effect free — twice for good measure. + zones_db.ZoneCatalogue(zones_schema).init_db().close() + zones_db.ZoneCatalogue(zones_schema).init_db().close() + + with pg_conn(zones_schema) as conn: + rows = {r["id"]: r["encounter_name"] for r in conn.execute("SELECT id, encounter_name FROM zone_encounters")} + assert rows[enc_id] == "Ire, Malevolence" # untouched — normalization retired + assert rows[enc_id2] == "Demetrius Crane" # untouched + + +def test_paren_zone_alias_resolves_to_canonical(zones_schema): + """Descendant of the SQLite " (Zone)" suffix-strip normalization test: + the rewrite itself was retired at cutover (prod data crossed over + already stripped, with the old suffixed name preserved as an alias) — + what survives is the alias contract: find_by_name resolves BOTH the + clean name and the historical parenthesised form to the same canonical + zone.""" + with pg_conn(zones_schema) as conn: conn.execute( - "INSERT INTO zones (name, name_lower, expansion_short, expansion_name, " + "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, " "expansion_confidence, expansion_source) " - "VALUES ('Kurn''s Tower (Zone)', 'kurn''s tower (zone)', 'TSO', 'The Shadow Odyssey', 'test', 'test')" + "VALUES (1, 'Kurn''s Tower', 'kurn''s tower', 'TSO', 'The Shadow Odyssey', 'test', 'test')" ) conn.execute( - "INSERT INTO zones (name, name_lower, expansion_short, expansion_name, " + "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, " "expansion_confidence, expansion_source) " - "VALUES ('Halls of Fate', 'halls of fate', 'EoF', 'Echoes of Faydwer', 'test', 'test')" + "VALUES (2, 'Halls of Fate', 'halls of fate', 'EoF', 'Echoes of Faydwer', 'test', 'test')" ) - conn.commit() - finally: - conn.close() - - # Re-init to trigger the cleanup. - conn = zones_db.ZoneCatalogue(p).init_db() - try: - names = {r[0]: r[1] for r in conn.execute("SELECT name, name_lower FROM zones ORDER BY name")} - assert "Kurn's Tower" in names, "suffix should be stripped" - assert names["Kurn's Tower"] == "kurn's tower", "name_lower should be stripped too" - assert "Halls of Fate" in names, "unaffected zone should remain unchanged" - assert "Kurn's Tower (Zone)" not in names, "old suffixed name should be gone" - - # Old name preserved as alias for backward-compat lookups. - alias_row = conn.execute( - "SELECT z.name FROM zone_aliases a JOIN zones z ON z.id = a.zone_id WHERE a.alias_lower = ?", - ("kurn's tower (zone)",), - ).fetchone() - assert alias_row is not None, "old name should be preserved as alias" - assert alias_row[0] == "Kurn's Tower", "alias should resolve to cleaned zone" - - # find_by_name resolves both forms to the same canonical zone. - from_clean = zones_db.ZoneCatalogue(p).find_by_name("Kurn's Tower") - from_suffixed = zones_db.ZoneCatalogue(p).find_by_name("Kurn's Tower (Zone)") - assert from_clean is not None and from_suffixed is not None - assert from_clean["name"] == from_suffixed["name"] == "Kurn's Tower" - finally: - conn.close() - - # Second run is a no-op (no rows match the LIKE filter anymore). - conn = zones_db.ZoneCatalogue(p).init_db() - try: - cleaned_name = conn.execute("SELECT name FROM zones WHERE name_lower = 'kurn''s tower'").fetchone() - assert cleaned_name is not None and cleaned_name[0] == "Kurn's Tower" - finally: - conn.close() - - -def _bootstrap_zone(p): + conn.execute( + "INSERT INTO zone_aliases (alias, alias_lower, zone_id) " + "VALUES ('Kurn''s Tower (Zone)', 'kurn''s tower (zone)', 1)" + ) + + cat = zones_db.ZoneCatalogue(zones_schema) + from_clean = cat.find_by_name("Kurn's Tower") + from_suffixed = cat.find_by_name("Kurn's Tower (Zone)") + assert from_clean is not None and from_suffixed is not None + assert from_clean["name"] == from_suffixed["name"] == "Kurn's Tower" + assert from_clean["name_lower"] == "kurn's tower" + # The unaffected zone resolves only by its own name. + assert cat.find_by_name("Halls of Fate") is not None + + +def _bootstrap_zone(schema: str) -> int: """Single zone + zero encounters. Returns zone_id.""" - conn = zones_db.ZoneCatalogue(p).init_db() - try: + with pg_conn(schema) as conn: conn.execute( - "INSERT INTO zones (name, name_lower, expansion_short, expansion_name, " + "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, " "expansion_confidence, expansion_source) " - "VALUES ('Test Zone', 'test zone', 'RoK', 'Rise of Kunark', 'test', 'test')" + "VALUES (1, 'Test Zone', 'test zone', 'RoK', 'Rise of Kunark', 'test', 'test')" ) - zid = conn.execute("SELECT id FROM zones WHERE name = 'Test Zone'").fetchone()[0] - conn.commit() - return zid - finally: - conn.close() + return 1 -def test_add_encounter_creates_row_and_position0_mob(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Adkar Vyx") +def test_add_encounter_creates_row_and_position0_mob(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Adkar Vyx") assert enc["encounter_name"] == "Adkar Vyx" assert enc["position"] == 1 assert enc["mobs"] == [{"mob_name": "Adkar Vyx", "position": 0}] -def test_add_encounter_appends_after_existing(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="First") - enc2 = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Second") +def test_add_encounter_appends_after_existing(zones_schema): + zid = _bootstrap_zone(zones_schema) + zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="First") + enc2 = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Second") assert enc2["position"] == 2 -def test_update_encounter_renames_primary_and_position0_mob(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Old Name") - updated = zones_db.ZoneCatalogue(p).update_encounter(enc["id"], primary_mob="New Name") +def test_update_encounter_renames_primary_and_position0_mob(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Old Name") + updated = zones_db.ZoneCatalogue(zones_schema).update_encounter(enc["id"], primary_mob="New Name") assert updated["encounter_name"] == "New Name" assert updated["mobs"][0]["mob_name"] == "New Name" -def test_update_encounter_stage_and_wiki(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Boss") - updated = zones_db.ZoneCatalogue(p).update_encounter(enc["id"], stage="Wing 1", wiki_url="http://x") +def test_update_encounter_stage_and_wiki(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Boss") + updated = zones_db.ZoneCatalogue(zones_schema).update_encounter(enc["id"], stage="Wing 1", wiki_url="http://x") assert updated["stage"] == "Wing 1" assert updated["wiki_url"] == "http://x" # primary unchanged assert updated["encounter_name"] == "Boss" -def test_delete_encounter_cascades_mobs(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Doomed") - assert zones_db.ZoneCatalogue(p).delete_encounter(enc["id"]) is True - with sqlite3.connect(p) as c: - assert c.execute("SELECT COUNT(*) FROM zone_encounters WHERE id = ?", (enc["id"],)).fetchone()[0] == 0 - assert ( - c.execute("SELECT COUNT(*) FROM zone_encounter_mobs WHERE encounter_id = ?", (enc["id"],)).fetchone()[0] - == 0 - ) +def test_delete_encounter_cascades_mobs(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Doomed") + assert zones_db.ZoneCatalogue(zones_schema).delete_encounter(enc["id"]) is True + with pg_conn(zones_schema) as c: + n_enc = c.execute("SELECT COUNT(*) AS n FROM zone_encounters WHERE id = %s", (enc["id"],)).fetchone()["n"] + n_mobs = c.execute( + "SELECT COUNT(*) AS n FROM zone_encounter_mobs WHERE encounter_id = %s", (enc["id"],) + ).fetchone()["n"] + assert n_enc == 0 + assert n_mobs == 0 -def test_delete_encounter_missing_returns_false(tmp_path): - p = tmp_path / "zones.db" - zones_db.ZoneCatalogue(p).init_db() - assert zones_db.ZoneCatalogue(p).delete_encounter(99999) is False +def test_delete_encounter_missing_returns_false(zones_schema): + assert zones_db.ZoneCatalogue(zones_schema).delete_encounter(99999) is False -def test_reorder_encounters_atomic_permutation(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - a = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="A") - b = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="B") - c = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="C") +def test_reorder_encounters_atomic_permutation(zones_schema): + zid = _bootstrap_zone(zones_schema) + a = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="A") + b = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="B") + c = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="C") # Reverse order: C, B, A - zones_db.ZoneCatalogue(p).reorder_encounters(zid, [c["id"], b["id"], a["id"]]) - with sqlite3.connect(p) as conn: + zones_db.ZoneCatalogue(zones_schema).reorder_encounters(zid, [c["id"], b["id"], a["id"]]) + with pg_conn(zones_schema) as conn: positions = { - r[0]: r[1] for r in conn.execute("SELECT id, position FROM zone_encounters WHERE zone_id = ?", (zid,)) + r["id"]: r["position"] + for r in conn.execute("SELECT id, position FROM zone_encounters WHERE zone_id = %s", (zid,)) } assert positions[c["id"]] == 1 assert positions[b["id"]] == 2 assert positions[a["id"]] == 3 -def test_reorder_encounters_rejects_missing_id(tmp_path): - import pytest as _pytest +def test_reorder_encounters_rejects_missing_id(zones_schema): + zid = _bootstrap_zone(zones_schema) + a = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="A") + b = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="B") + with pytest.raises(ValueError): + zones_db.ZoneCatalogue(zones_schema).reorder_encounters(zid, [a["id"]]) # missing b + with pytest.raises(ValueError): + zones_db.ZoneCatalogue(zones_schema).reorder_encounters(zid, [a["id"], b["id"], 9999]) # extra - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - a = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="A") - b = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="B") - with _pytest.raises(ValueError): - zones_db.ZoneCatalogue(p).reorder_encounters(zid, [a["id"]]) # missing b - with _pytest.raises(ValueError): - zones_db.ZoneCatalogue(p).reorder_encounters(zid, [a["id"], b["id"], 9999]) # extra +def test_reorder_encounters_rejects_duplicates(zones_schema): + zid = _bootstrap_zone(zones_schema) + a = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="A") + b = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="B") + with pytest.raises(ValueError): + zones_db.ZoneCatalogue(zones_schema).reorder_encounters(zid, [a["id"], a["id"], b["id"]]) -def test_reorder_encounters_rejects_duplicates(tmp_path): - import pytest as _pytest - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - a = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="A") - b = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="B") - with _pytest.raises(ValueError): - zones_db.ZoneCatalogue(p).reorder_encounters(zid, [a["id"], a["id"], b["id"]]) - - -def test_add_mob_appends_sibling(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Primary") - sib = zones_db.ZoneCatalogue(p).add_mob(enc["id"], mob_name="Sibling") +def test_add_mob_appends_sibling(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Primary") + sib = zones_db.ZoneCatalogue(zones_schema).add_mob(enc["id"], mob_name="Sibling") assert sib["position"] == 1 - enc2 = zones_db.ZoneCatalogue(p).add_mob(enc["id"], mob_name="Third") + enc2 = zones_db.ZoneCatalogue(zones_schema).add_mob(enc["id"], mob_name="Third") assert enc2["position"] == 2 -def test_add_mob_make_primary_shifts_old_primary(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="OldPrimary") - zones_db.ZoneCatalogue(p).add_mob(enc["id"], mob_name="NewPrimary", make_primary=True) - with sqlite3.connect(p) as conn: +def test_add_mob_make_primary_shifts_old_primary(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="OldPrimary") + zones_db.ZoneCatalogue(zones_schema).add_mob(enc["id"], mob_name="NewPrimary", make_primary=True) + with pg_conn(zones_schema) as conn: mobs = [ - (r[0], r[1]) + (r["mob_name"], r["position"]) for r in conn.execute( - "SELECT mob_name, position FROM zone_encounter_mobs WHERE encounter_id = ? ORDER BY position", + "SELECT mob_name, position FROM zone_encounter_mobs WHERE encounter_id = %s ORDER BY position", (enc["id"],), ) ] + name = conn.execute("SELECT encounter_name FROM zone_encounters WHERE id = %s", (enc["id"],)).fetchone()[ + "encounter_name" + ] assert mobs[0] == ("NewPrimary", 0) assert ("OldPrimary", 1) in mobs - with sqlite3.connect(p) as conn: - name = conn.execute("SELECT encounter_name FROM zone_encounters WHERE id = ?", (enc["id"],)).fetchone()[0] assert name == "NewPrimary" -def test_update_mob_renames_primary_updates_encounter_name(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Primary") - sib = zones_db.ZoneCatalogue(p).add_mob(enc["id"], mob_name="Sibling") - primary_id = next(m["id"] for m in zones_db.ZoneCatalogue(p).list_mobs(enc["id"]) if m["position"] == 0) - zones_db.ZoneCatalogue(p).update_mob(primary_id, mob_name="Renamed") - with sqlite3.connect(p) as conn: +def test_update_mob_renames_primary_updates_encounter_name(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Primary") + sib = zones_db.ZoneCatalogue(zones_schema).add_mob(enc["id"], mob_name="Sibling") + primary_id = next(m["id"] for m in zones_db.ZoneCatalogue(zones_schema).list_mobs(enc["id"]) if m["position"] == 0) + zones_db.ZoneCatalogue(zones_schema).update_mob(primary_id, mob_name="Renamed") + with pg_conn(zones_schema) as conn: assert ( conn.execute( - "SELECT encounter_name FROM zone_encounters WHERE id = ?", + "SELECT encounter_name FROM zone_encounters WHERE id = %s", (enc["id"],), - ).fetchone()[0] + ).fetchone()["encounter_name"] == "Renamed" ) # Renaming a sibling must NOT touch encounter_name - zones_db.ZoneCatalogue(p).update_mob(sib["id"], mob_name="SibRenamed") - with sqlite3.connect(p) as conn: + zones_db.ZoneCatalogue(zones_schema).update_mob(sib["id"], mob_name="SibRenamed") + with pg_conn(zones_schema) as conn: assert ( conn.execute( - "SELECT encounter_name FROM zone_encounters WHERE id = ?", + "SELECT encounter_name FROM zone_encounters WHERE id = %s", (enc["id"],), - ).fetchone()[0] + ).fetchone()["encounter_name"] == "Renamed" ) -def test_promote_mob_swaps_with_primary(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Primary") - sib = zones_db.ZoneCatalogue(p).add_mob(enc["id"], mob_name="Sibling") - zones_db.ZoneCatalogue(p).promote_mob(sib["id"]) - with sqlite3.connect(p) as conn: +def test_promote_mob_swaps_with_primary(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Primary") + sib = zones_db.ZoneCatalogue(zones_schema).add_mob(enc["id"], mob_name="Sibling") + zones_db.ZoneCatalogue(zones_schema).promote_mob(sib["id"]) + with pg_conn(zones_schema) as conn: mobs = [ - (r[0], r[1]) + (r["mob_name"], r["position"]) for r in conn.execute( - "SELECT mob_name, position FROM zone_encounter_mobs WHERE encounter_id = ? ORDER BY position", + "SELECT mob_name, position FROM zone_encounter_mobs WHERE encounter_id = %s ORDER BY position", (enc["id"],), ) ] - name = conn.execute("SELECT encounter_name FROM zone_encounters WHERE id = ?", (enc["id"],)).fetchone()[0] + name = conn.execute("SELECT encounter_name FROM zone_encounters WHERE id = %s", (enc["id"],)).fetchone()[ + "encounter_name" + ] assert mobs == [("Sibling", 0), ("Primary", 1)] assert name == "Sibling" -def test_promote_mob_noop_when_already_primary(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="OnlyOne") - primary_id = next(m["id"] for m in zones_db.ZoneCatalogue(p).list_mobs(enc["id"]) if m["position"] == 0) - result = zones_db.ZoneCatalogue(p).promote_mob(primary_id) +def test_promote_mob_noop_when_already_primary(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="OnlyOne") + primary_id = next(m["id"] for m in zones_db.ZoneCatalogue(zones_schema).list_mobs(enc["id"]) if m["position"] == 0) + result = zones_db.ZoneCatalogue(zones_schema).promote_mob(primary_id) assert result["position"] == 0 assert result["mob_name"] == "OnlyOne" -def test_delete_mob_refuses_last_mob(tmp_path): - import pytest as _pytest - - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Only") - only_id = next(m["id"] for m in zones_db.ZoneCatalogue(p).list_mobs(enc["id"]) if m["position"] == 0) - with _pytest.raises(ValueError, match="last mob"): - zones_db.ZoneCatalogue(p).delete_mob(only_id) - +def test_delete_mob_refuses_last_mob(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Only") + only_id = next(m["id"] for m in zones_db.ZoneCatalogue(zones_schema).list_mobs(enc["id"]) if m["position"] == 0) + with pytest.raises(ValueError, match="last mob"): + zones_db.ZoneCatalogue(zones_schema).delete_mob(only_id) -def test_delete_mob_refuses_primary_while_siblings_exist(tmp_path): - import pytest as _pytest - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Primary") - zones_db.ZoneCatalogue(p).add_mob(enc["id"], mob_name="Sibling") - primary_id = next(m["id"] for m in zones_db.ZoneCatalogue(p).list_mobs(enc["id"]) if m["position"] == 0) - with _pytest.raises(ValueError, match="primary"): - zones_db.ZoneCatalogue(p).delete_mob(primary_id) +def test_delete_mob_refuses_primary_while_siblings_exist(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Primary") + zones_db.ZoneCatalogue(zones_schema).add_mob(enc["id"], mob_name="Sibling") + primary_id = next(m["id"] for m in zones_db.ZoneCatalogue(zones_schema).list_mobs(enc["id"]) if m["position"] == 0) + with pytest.raises(ValueError, match="primary"): + zones_db.ZoneCatalogue(zones_schema).delete_mob(primary_id) -def test_delete_mob_sibling_succeeds(tmp_path): - p = tmp_path / "zones.db" - zid = _bootstrap_zone(p) - enc = zones_db.ZoneCatalogue(p).add_encounter(zid, primary_mob="Primary") - sib = zones_db.ZoneCatalogue(p).add_mob(enc["id"], mob_name="Sibling") - assert zones_db.ZoneCatalogue(p).delete_mob(sib["id"]) is True - mobs = zones_db.ZoneCatalogue(p).list_mobs(enc["id"]) +def test_delete_mob_sibling_succeeds(zones_schema): + zid = _bootstrap_zone(zones_schema) + enc = zones_db.ZoneCatalogue(zones_schema).add_encounter(zid, primary_mob="Primary") + sib = zones_db.ZoneCatalogue(zones_schema).add_mob(enc["id"], mob_name="Sibling") + assert zones_db.ZoneCatalogue(zones_schema).delete_mob(sib["id"]) is True + mobs = zones_db.ZoneCatalogue(zones_schema).list_mobs(enc["id"]) assert [m["mob_name"] for m in mobs] == ["Primary"] -def test_delete_mob_missing_returns_false(tmp_path): - p = tmp_path / "zones.db" - zones_db.ZoneCatalogue(p).init_db() - assert zones_db.ZoneCatalogue(p).delete_mob(99999) is False +def test_delete_mob_missing_returns_false(zones_schema): + assert zones_db.ZoneCatalogue(zones_schema).delete_mob(99999) is False diff --git a/tests/server/test_act_categories.py b/tests/server/test_act_categories.py index 9d34fb73..6ac6bba1 100644 --- a/tests/server/test_act_categories.py +++ b/tests/server/test_act_categories.py @@ -3,8 +3,8 @@ Categories are synthetic encounters under the "General" raid zone, so the existing per-encounter trigger routes address them via /api/zones/General/encounters/{position}/... — covered here end-to-end -against a real tmp raids.db (no resolution mocking: the General branch in -_resolve_encounter_sync bypasses zones_db entirely).""" +against a real leased raids schema (no resolution mocking: the General +branch in _resolve_encounter_sync bypasses zones_db entirely).""" from __future__ import annotations @@ -13,17 +13,12 @@ @pytest.fixture -def raids_tmp(tmp_path, monkeypatch): - """Point the raids catalogue at a fresh tmp DB for each test.""" - from backend.eq2db.raids import catalogue - from backend.server.api.act import _shared - - db_file = tmp_path / "raids.db" - monkeypatch.setattr(catalogue, "path", db_file) - # The module-level "already inited" flag would skip schema creation on - # the fresh file. - monkeypatch.setattr(_shared, "_RAIDS_DB_INIT_DONE", False) - return db_file +def raids_tmp(raids_schema: str) -> str: + """Isolated leased raids schema per test (conftest ``raids_schema``) — + the shared catalogue is already re-pointed at it, which covers the act + routes' ``raids_db.init_db()`` checkouts. Aliased so the tests can keep + naming it ``raids_tmp``.""" + return raids_schema def _writer_client(app): diff --git a/tests/server/test_act_pack.py b/tests/server/test_act_pack.py index 9bf16735..cb6bce18 100644 --- a/tests/server/test_act_pack.py +++ b/tests/server/test_act_pack.py @@ -41,14 +41,13 @@ def test_control_effect_rejects_unknown(self): @pytest.fixture -def seeded_raids_db(tmp_path, monkeypatch): - """A raids.db with one curated encounter (trigger + timer, enriched) and - one bare encounter (no ACT content — must be excluded from the pack), - swapped in as the shared catalogue's path.""" +def seeded_raids_db(raids_schema): + """A leased raids schema with one curated encounter (trigger + timer, + enriched) and one bare encounter (no ACT content — must be excluded from + the pack). The shared catalogue is already re-pointed at the schema.""" from backend.eq2db import raids as raids_module - p = tmp_path / "raids.db" - conn = RaidCatalogue(p).init_db() + conn = raids_module.catalogue.init_db() zone_id = raids_module.catalogue.upsert_raid_zone( conn, zone_name="Freethinker Hideout", expansion_short="EoF", source="manual" ) @@ -77,9 +76,7 @@ def seeded_raids_db(tmp_path, monkeypatch): damage_type="crushing", ) conn.close() - - monkeypatch.setattr(raids_module.catalogue, "path", p) - return p + return raids_schema @pytest.mark.asyncio @@ -118,12 +115,12 @@ async def test_pack_version_changes_on_edit(app, seeded_raids_db): from backend.eq2db.raids import catalogue conn = RaidCatalogue(seeded_raids_db).init_db() - enc = conn.execute("SELECT id FROM raid_encounters WHERE mob_name = 'Malkonis D''Morte'").fetchone()[0] + enc = conn.execute("SELECT id FROM raid_encounters WHERE mob_name = 'Malkonis D''Morte'").fetchone()["id"] RaidCatalogue.upsert_act_trigger( conn, raid_encounter_id=enc, regex="Come out and join me my brethren!", sound_data="Adds" ) conn.close() - assert catalogue.path == seeded_raids_db # the endpoint reads the seeded file + assert catalogue.schema == seeded_raids_db # the endpoint reads the seeded schema async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: after = (await client.get("/api/act/pack")).json()["version"] diff --git a/tests/server/test_act_triggers.py b/tests/server/test_act_triggers.py index fe73458b..321c4b46 100644 --- a/tests/server/test_act_triggers.py +++ b/tests/server/test_act_triggers.py @@ -15,9 +15,9 @@ from __future__ import annotations -import sqlite3 from unittest.mock import patch +import psycopg.errors import pytest from httpx import ASGITransport, AsyncClient @@ -372,7 +372,7 @@ async def test_create_spell_timer_name_collision_is_409(app): patch("backend.server.api.act.spell_timers.raids_db.init_db") as m_init, patch( "backend.server.api.act.spell_timers.raids_db.upsert_act_spell_timer", - side_effect=sqlite3.IntegrityError("UNIQUE constraint failed"), + side_effect=psycopg.errors.UniqueViolation("duplicate key value violates unique constraint"), ), ): m_init.return_value.close = lambda: None diff --git a/tests/server/test_raid_bosses.py b/tests/server/test_raid_bosses.py index 63424568..24c22b13 100644 --- a/tests/server/test_raid_bosses.py +++ b/tests/server/test_raid_bosses.py @@ -3,58 +3,57 @@ from __future__ import annotations -import sqlite3 -from pathlib import Path from unittest.mock import patch import pytest from httpx import ASGITransport, AsyncClient -from backend.eq2db.zones import catalogue as zones_db from backend.server.api import rankings - -# Only the columns the query touches — the real schema has many more. -_MINI_SCHEMA = """ -CREATE TABLE zones (id INTEGER PRIMARY KEY); -CREATE TABLE zone_types (zone_id INTEGER, type TEXT); -CREATE TABLE zone_encounters (id INTEGER PRIMARY KEY, zone_id INTEGER); -CREATE TABLE zone_encounter_mobs (encounter_id INTEGER, mob_name_lower TEXT); -""" - - -def _seed(path: Path) -> None: - conn = sqlite3.connect(path) - try: - conn.executescript(_MINI_SCHEMA) - conn.executemany("INSERT INTO zones VALUES (?)", [(1,), (2,), (3,)]) - conn.executemany( - "INSERT INTO zone_types VALUES (?, ?)", +from tests.fixtures.pg import pg_conn + + +def _seed(schema: str) -> None: + """Seed the leased zones schema with two raid zones, one dungeon, and + their encounter/mob rosters. Explicit encounter ids need OVERRIDING + SYSTEM VALUE (identity column); positions satisfy UNIQUE(zone_id, + position).""" + with pg_conn(schema) as conn: + conn.cursor().executemany( + "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, expansion_confidence) " + "VALUES (%s, %s, %s, %s, %s, 'category')", + [ + (1, "Raid Zone X4", "raid zone x4", "RoK", "Rise of Kunark"), + (2, "Raid Zone X2", "raid zone x2", "RoK", "Rise of Kunark"), + (3, "Dungeon Zone", "dungeon zone", "RoK", "Rise of Kunark"), + ], + ) + conn.cursor().executemany( + "INSERT INTO zone_types (zone_id, type) VALUES (%s, %s)", [(1, "raid_x4"), (2, "raid_x2"), (3, "dungeon"), (1, "instance")], ) - conn.executemany("INSERT INTO zone_encounters VALUES (?, ?)", [(10, 1), (11, 1), (20, 2), (30, 3)]) - conn.executemany( - "INSERT INTO zone_encounter_mobs VALUES (?, ?)", + conn.cursor().executemany( + "INSERT INTO zone_encounters (id, zone_id, encounter_name, position) " + "OVERRIDING SYSTEM VALUE VALUES (%s, %s, %s, %s)", + [(10, 1, "Trakanon", 1), (11, 1, "Hoshkar", 2), (20, 2, "Herald", 1), (30, 3, "Dungeon Boss", 1)], + ) + conn.cursor().executemany( + "INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower) VALUES (%s, %s, %s)", [ - (10, "trakanon"), - (11, "hoshkar"), - (11, "Silverwing"), # curator typo in case — normalised - (20, "the herald of wuoshi’s"), # curly apostrophe → ' - (20, "trakanon"), # duplicate across zones collapses - (30, "dungeon boss"), # not a raid zone → excluded + (10, "trakanon", "trakanon"), + (11, "hoshkar", "hoshkar"), + (11, "Silverwing", "Silverwing"), # curator typo in case — normalised + (20, "the herald of wuoshi’s", "the herald of wuoshi’s"), # curly apostrophe → ' + (20, "trakanon", "trakanon"), # duplicate across zones collapses + (30, "dungeon boss", "dungeon boss"), # not a raid zone → excluded ], ) - conn.commit() - finally: - conn.close() @pytest.fixture -def raid_db(tmp_path, monkeypatch): - db = tmp_path / "zones.db" - _seed(db) - monkeypatch.setattr(zones_db, "path", db) +def raid_db(zones_schema): + _seed(zones_schema) rankings._raid_boss_names.cache_clear() - yield db + yield zones_schema rankings._raid_boss_names.cache_clear() @@ -62,8 +61,8 @@ def test_names_are_raid_only_normalised_deduped_and_sorted(raid_db): assert rankings._raid_boss_names() == ("hoshkar", "silverwing", "the herald of wuoshi's", "trakanon") -def test_missing_zones_db_is_empty(tmp_path, monkeypatch): - monkeypatch.setattr(zones_db, "path", tmp_path / "nope.db") +def test_unseeded_zones_schema_is_empty(zones_schema): + """No curated rosters at all (the old missing-zones.db case) → empty.""" rankings._raid_boss_names.cache_clear() try: assert rankings._raid_boss_names() == () @@ -77,10 +76,11 @@ def test_version_is_a_content_hash(raid_db): assert first["version"] == rankings.raid_boss_pack()["version"] assert len(first["version"]) == 12 # A roster edit changes the stamp once the cache is invalidated. - conn = sqlite3.connect(raid_db) - conn.execute("INSERT INTO zone_encounter_mobs VALUES (10, 'new boss')") - conn.commit() - conn.close() + with pg_conn(raid_db) as conn: + conn.execute( + "INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower) " + "VALUES (10, 'new boss', 'new boss')" + ) assert rankings.raid_boss_pack()["version"] == first["version"] # cached rankings.invalidate_zones_cache() second = rankings.raid_boss_pack() diff --git a/tests/server/test_raid_curation.py b/tests/server/test_raid_curation.py index 4db9664d..c047d433 100644 --- a/tests/server/test_raid_curation.py +++ b/tests/server/test_raid_curation.py @@ -12,7 +12,7 @@ Most tests mock the zones_db helpers and assert the route layer's auth + HTTP-status behaviour. The bottom section runs real end-to-end roundtrips -against a throwaway zones.db to verify the SQL contracts. +against a leased scratch zones schema to verify the SQL contracts. """ from __future__ import annotations @@ -23,6 +23,7 @@ from httpx import ASGITransport, AsyncClient from backend.server.auth_deps import require_admin +from tests.fixtures.pg import pg_conn # A representative hydrated-zone shape — matches what the real helpers return. _ZONE = { @@ -242,16 +243,13 @@ async def test_delete_zone_happy_path(app, admin_override): # ── Real-helper roundtrip tests (no SQL mocks) ──────────────────────────────── -def _seed_zones(db_path): - """Insert a representative set of zones into a fresh zones.db.""" - from backend.eq2db import zones as zdb - - conn = zdb.ZoneCatalogue(db_path).init_db() - try: +def _seed_zones(schema): + """Insert a representative set of zones into a fresh zones schema.""" + with pg_conn(schema) as conn: # RoK raid + RoK non-raid + EoF raid (raid_x2) + TSO raid. - conn.executemany( + conn.cursor().executemany( "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, " - "expansion_year, expansion_confidence) VALUES (?, ?, ?, ?, ?, ?, 'category')", + "expansion_year, expansion_confidence) VALUES (%s, %s, %s, %s, %s, %s, 'category')", [ (1, "Veeshan's Peak", "veeshan's peak", "RoK", "Rise of Kunark", 2007), (2, "Karnor's Castle", "karnor's castle", "RoK", "Rise of Kunark", 2007), @@ -259,23 +257,20 @@ def _seed_zones(db_path): (4, "Munzok's Material Bastion", "munzok's material bastion", "TSO", "The Shadow Odyssey", 2008), ], ) - conn.executemany( - "INSERT INTO zone_types (zone_id, type) VALUES (?, ?)", + conn.cursor().executemany( + "INSERT INTO zone_types (zone_id, type) VALUES (%s, %s)", [ (1, "raid_x4"), (3, "raid_x2"), (4, "raid_x4"), ], ) - conn.commit() - finally: - conn.close() -def test_add_then_list_then_remove_expansion_roundtrip(tmp_path): +def test_add_then_list_then_remove_expansion_roundtrip(zones_schema): from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) # Initially nothing featured. @@ -303,11 +298,11 @@ def test_add_then_list_then_remove_expansion_roundtrip(tmp_path): assert zdb.ZoneCatalogue(db).remove_featured_raid_expansion("TSO") is False -def test_add_zone_validates_raid_type(tmp_path): +def test_add_zone_validates_raid_type(zones_schema): """Only raid_x4/raid_x2-tagged zones can be featured.""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) # raid_x4 → OK @@ -326,13 +321,13 @@ def test_add_zone_validates_raid_type(tmp_path): assert zdb.ZoneCatalogue(db).add_featured_raid_zone("Imaginary") is None -def test_featured_zones_imply_expansion_in_list(tmp_path): +def test_featured_zones_imply_expansion_in_list(zones_schema): """Adding a featured zone whose expansion isn't explicitly featured must still surface that expansion in list_featured_raid_expansions (the implicit case).""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) # No featured_raid_expansions rows, but feature a RoK raid zone. @@ -347,24 +342,24 @@ def test_featured_zones_imply_expansion_in_list(tmp_path): assert "RoK" not in available -def test_remove_expansion_cascades_to_featured_zones(tmp_path): +def test_remove_expansion_cascades_to_featured_zones(zones_schema): """Removing an expansion wipes its featured_raid_zones rows, but the underlying zone_encounters data is left intact.""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) # Seed a curator encounter for Veeshan's Peak so we can verify it survives. - import sqlite3 - - with sqlite3.connect(db) as conn: - conn.execute("INSERT INTO zone_encounters (id, zone_id, encounter_name, position) VALUES (10, 1, 'Druushk', 1)") + with pg_conn(db) as conn: + conn.execute( + "INSERT INTO zone_encounters (id, zone_id, encounter_name, position) " + "OVERRIDING SYSTEM VALUE VALUES (10, 1, 'Druushk', 1)" + ) conn.execute( "INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) " "VALUES (10, 'Druushk', 'druushk', 0)" ) - conn.commit() # Feature the expansion + zone. assert zdb.ZoneCatalogue(db).add_featured_raid_expansion("RoK") is True @@ -376,18 +371,18 @@ def test_remove_expansion_cascades_to_featured_zones(tmp_path): assert zdb.ZoneCatalogue(db).list_featured_raid_zones("RoK") == [] # But the underlying zone_encounters / mobs row is preserved. - with sqlite3.connect(db) as conn: - n = conn.execute("SELECT COUNT(*) FROM zone_encounters WHERE id = 10").fetchone()[0] - m = conn.execute("SELECT COUNT(*) FROM zone_encounter_mobs WHERE encounter_id = 10").fetchone()[0] + with pg_conn(db) as conn: + n = conn.execute("SELECT COUNT(*) AS n FROM zone_encounters WHERE id = 10").fetchone()["n"] + m = conn.execute("SELECT COUNT(*) AS n FROM zone_encounter_mobs WHERE encounter_id = 10").fetchone()["n"] assert n == 1 assert m == 1 -def test_available_zones_excludes_featured(tmp_path): +def test_available_zones_excludes_featured(zones_schema): """Once a zone is featured, it's no longer offered in the 'add zone' picker.""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) before = {z["name"] for z in zdb.ZoneCatalogue(db).list_available_raid_zones("RoK")} @@ -399,73 +394,67 @@ def test_available_zones_excludes_featured(tmp_path): assert "Veeshan's Peak" not in after -def test_remove_zone_preserves_encounters(tmp_path): +def test_remove_zone_preserves_encounters(zones_schema): """remove_featured_raid_zone wipes the featured row only — boss data in zone_encounters stays untouched.""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) - import sqlite3 - - with sqlite3.connect(db) as conn: + with pg_conn(db) as conn: conn.execute( - "INSERT INTO zone_encounters (id, zone_id, encounter_name, position) VALUES (11, 1, 'Phara Dar', 1)" + "INSERT INTO zone_encounters (id, zone_id, encounter_name, position) " + "OVERRIDING SYSTEM VALUE VALUES (11, 1, 'Phara Dar', 1)" ) conn.execute( "INSERT INTO zone_encounter_mobs (encounter_id, mob_name, mob_name_lower, position) " "VALUES (11, 'Phara Dar', 'phara dar', 0)" ) - conn.commit() assert zdb.ZoneCatalogue(db).add_featured_raid_zone("Veeshan's Peak") is not None assert zdb.ZoneCatalogue(db).remove_featured_raid_zone("Veeshan's Peak") is True - with sqlite3.connect(db) as conn: - n = conn.execute("SELECT COUNT(*) FROM zone_encounters WHERE id = 11").fetchone()[0] + with pg_conn(db) as conn: + n = conn.execute("SELECT COUNT(*) AS n FROM zone_encounters WHERE id = 11").fetchone()["n"] assert n == 1 # ── Categories + drag-reorder ──────────────────────────────────────────────── -def _seed_multi_raid_expansion(db_path): - """Seed a zones.db with four raid_x4 zones in the RoK expansion. All four - are featured under the (NULL = "Uncategorised") lane so the reorder +def _seed_multi_raid_expansion(schema): + """Seed a zones schema with four raid_x4 zones in the RoK expansion. All + four are featured under the (NULL = "Uncategorised") lane so the reorder tests have a baseline to permute.""" from backend.eq2db import zones as zdb - _seed_zones(db_path) + _seed_zones(schema) # Add three more RoK raid_x4 zones so we have 4 to permute. - conn = zdb.ZoneCatalogue(db_path).init_db() - try: - conn.executemany( + with pg_conn(schema) as conn: + conn.cursor().executemany( "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, " - "expansion_year, expansion_confidence) VALUES (?, ?, ?, ?, ?, ?, 'category')", + "expansion_year, expansion_confidence) VALUES (%s, %s, %s, %s, %s, %s, 'category')", [ (5, "Trakanon's Lair", "trakanon's lair", "RoK", "Rise of Kunark", 2007), (6, "Chardok: The Bloodied Halls", "chardok: the bloodied halls", "RoK", "Rise of Kunark", 2007), (7, "Sebilis", "sebilis", "RoK", "Rise of Kunark", 2007), ], ) - conn.executemany( - "INSERT INTO zone_types (zone_id, type) VALUES (?, ?)", + conn.cursor().executemany( + "INSERT INTO zone_types (zone_id, type) VALUES (%s, %s)", [(5, "raid_x4"), (6, "raid_x4"), (7, "raid_x4")], ) - conn.commit() - finally: - conn.close() # Feature all four (each lands at next position in the NULL lane). for n in ("Veeshan's Peak", "Trakanon's Lair", "Chardok: The Bloodied Halls", "Sebilis"): - assert zdb.ZoneCatalogue(db_path).add_featured_raid_zone(n) is not None + assert zdb.ZoneCatalogue(schema).add_featured_raid_zone(n) is not None -def test_add_featured_raid_zone_assigns_increasing_positions_in_null_lane(tmp_path): +def test_add_featured_raid_zone_assigns_increasing_positions_in_null_lane(zones_schema): """First add → position 0, second → 1, third → 2. All in the NULL lane.""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_multi_raid_expansion(db) rows = zdb.ZoneCatalogue(db).list_featured_raid_zones("RoK") positions = [(r["name"], r["position"], r["category"]) for r in rows] @@ -478,12 +467,12 @@ def test_add_featured_raid_zone_assigns_increasing_positions_in_null_lane(tmp_pa ] -def test_list_featured_raid_zones_sorted_by_category_then_position(tmp_path): +def test_list_featured_raid_zones_sorted_by_category_then_position(zones_schema): """After categorising a subset, list returns NULL-lane first then categorised lanes ordered by (category, position).""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_multi_raid_expansion(db) # Put VP + Chardok in "Wing A" (positions 0, 1), leave the rest NULL. @@ -508,12 +497,12 @@ def test_list_featured_raid_zones_sorted_by_category_then_position(tmp_path): ] -def test_reorder_zones_autocreates_missing_categories(tmp_path): +def test_reorder_zones_autocreates_missing_categories(zones_schema): """A category name that appears in `ordering` but isn't yet tracked in featured_raid_categories is inserted at MAX(position)+1.""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_multi_raid_expansion(db) assert zdb.ZoneCatalogue(db).list_featured_raid_categories("RoK") == [] @@ -535,12 +524,12 @@ def test_reorder_zones_autocreates_missing_categories(tmp_path): assert positions[0] != positions[1] -def test_reorder_zones_returns_false_for_unfeatured_zone(tmp_path): +def test_reorder_zones_returns_false_for_unfeatured_zone(zones_schema): """A zone in `ordering` that isn't in featured_raid_zones for the expansion → reorder returns False (route layer maps to 400).""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_multi_raid_expansion(db) ok = zdb.ZoneCatalogue(db).reorder_featured_raid_zones( @@ -550,12 +539,12 @@ def test_reorder_zones_returns_false_for_unfeatured_zone(tmp_path): assert ok is False -def test_reorder_categories_atomic_two_phase(tmp_path): +def test_reorder_categories_atomic_two_phase(zones_schema): """Reordering categories rewrites position via the two-phase pattern — no UNIQUE/ordering collision possible even when swapping positions.""" from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_multi_raid_expansion(db) # Create three categories at positions 0, 1, 2. @@ -585,10 +574,10 @@ def test_reorder_categories_atomic_two_phase(tmp_path): assert [c["name"] for c in cats_after] == ["C", "B", "A"] -def test_reorder_categories_returns_false_for_missing_category(tmp_path): +def test_reorder_categories_returns_false_for_missing_category(zones_schema): from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_multi_raid_expansion(db) ok = zdb.ZoneCatalogue(db).reorder_featured_raid_categories( "RoK", @@ -841,10 +830,10 @@ async def test_delete_category_nonexistent_returns_removed_false(app, admin_over # ── Real-helper roundtrip tests for create + delete category ───────────────── -def test_create_featured_raid_category_roundtrip(tmp_path): +def test_create_featured_raid_category_roundtrip(zones_schema): from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) assert zdb.ZoneCatalogue(db).add_featured_raid_expansion("RoK") is True @@ -864,10 +853,10 @@ def test_create_featured_raid_category_roundtrip(tmp_path): assert tier2_pos > tier1_pos -def test_delete_featured_raid_category_moves_zones_to_null(tmp_path): +def test_delete_featured_raid_category_moves_zones_to_null(zones_schema): from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_multi_raid_expansion(db) # Put VP in "Wing A". @@ -895,10 +884,9 @@ def test_delete_featured_raid_category_moves_zones_to_null(tmp_path): assert zdb.ZoneCatalogue(db).delete_featured_raid_category("RoK", "Wing A") is False -def test_delete_featured_raid_category_nonexistent_returns_false(tmp_path): +def test_delete_featured_raid_category_nonexistent_returns_false(zones_schema): from backend.eq2db import zones as zdb - db = tmp_path / "zones.db" + db = zones_schema _seed_zones(db) - zdb.ZoneCatalogue(db).init_db() assert zdb.ZoneCatalogue(db).delete_featured_raid_category("RoK", "DoesNotExist") is False diff --git a/tests/server/test_zone_overview_revisions.py b/tests/server/test_zone_overview_revisions.py index c854cdd6..12edba55 100644 --- a/tests/server/test_zone_overview_revisions.py +++ b/tests/server/test_zone_overview_revisions.py @@ -3,17 +3,11 @@ from __future__ import annotations -import os -import shutil -import sqlite3 -import tempfile -from pathlib import Path from unittest.mock import patch import pytest from httpx import ASGITransport, AsyncClient -from backend.eq2db import raids as raids_db from backend.server import db as users_db # --------------------------------------------------------------------------- @@ -41,16 +35,12 @@ def _writer_client(app): @pytest.fixture() -def raids_tmp(tmp_path): - """Point raid_strategies at a real RaidCatalogue over a temp raids.db. - - The catalogue is a full drop-in for the route module's ``raids_db`` - binding (init_db uses the instance path; SOURCE_* are class attributes; - the write helpers are staticmethods). Returns the Path so individual - tests can open it for inspection.""" - db_path = tmp_path / "raids_test.db" - with patch("backend.server.api.raid_strategies.raids_db", raids_db.RaidCatalogue(db_path)): - yield db_path +def raids_tmp(raids_schema: str) -> str: + """Isolated leased raids schema (conftest ``raids_schema``) — the shared + catalogue the raid_strategies routes bind as ``raids_db`` is already + re-pointed at it, so no route-module patch is needed. Aliased so the + tests can keep naming it ``raids_tmp``.""" + return raids_schema # --------------------------------------------------------------------------- @@ -61,8 +51,6 @@ def raids_tmp(tmp_path): @pytest.mark.asyncio async def test_first_overview_write_creates_revision_with_null_before(app, raids_tmp): """PUT a brand-new overview → GET revisions shows one row with before_md=None.""" - raids_db.RaidCatalogue(raids_tmp).init_db().close() - with patch("backend.server.api.raid_strategies.zones_db.find_by_name", return_value=_fake_zone()): async with _writer_client(app) as client: put_r = await client.put( @@ -85,8 +73,6 @@ async def test_first_overview_write_creates_revision_with_null_before(app, raids @pytest.mark.asyncio async def test_overview_update_creates_revision_with_before_and_after(app, raids_tmp): """PUT twice with different markdown → two revision rows newest-first.""" - raids_db.RaidCatalogue(raids_tmp).init_db().close() - with patch("backend.server.api.raid_strategies.zones_db.find_by_name", return_value=_fake_zone()): async with _writer_client(app) as client: await client.put( @@ -114,8 +100,6 @@ async def test_overview_update_creates_revision_with_before_and_after(app, raids @pytest.mark.asyncio async def test_overview_unchanged_skips_revision(app, raids_tmp): """PUT the same markdown twice → only ONE revision row (no duplicate).""" - raids_db.RaidCatalogue(raids_tmp).init_db().close() - same_md = "## tactics unchanged" with patch("backend.server.api.raid_strategies.zones_db.find_by_name", return_value=_fake_zone()): async with _writer_client(app) as client: @@ -140,8 +124,6 @@ async def test_revisions_endpoint_404_unknown_zone(app): @pytest.mark.asyncio async def test_revisions_endpoint_returns_empty_when_no_overview_written(app, raids_tmp): """Zone exists in zones.db but no overview PUT yet → 200 with empty revisions list.""" - raids_db.RaidCatalogue(raids_tmp).init_db().close() - with patch("backend.server.api.raid_strategies.zones_db.find_by_name", return_value=_fake_zone()): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: r = await client.get("/api/zones/The Emerald Halls/overview/revisions") @@ -151,10 +133,8 @@ async def test_revisions_endpoint_returns_empty_when_no_overview_written(app, ra @pytest.mark.asyncio -async def test_revisions_endpoint_returns_editor_display_name(app, raids_tmp): +async def test_revisions_endpoint_returns_editor_display_name(app, raids_tmp, users_schema): """PUT overview as a known user → GET revisions shows edited_by_name.""" - raids_db.RaidCatalogue(raids_tmp).init_db().close() - # Seed the user so get_display_names_for_discord_ids resolves. await users_db.upsert_user( discord_id="known-editor-1", diff --git a/tests/server/test_zones_admin_types.py b/tests/server/test_zones_admin_types.py index 04f72fb5..6fbb8a14 100644 --- a/tests/server/test_zones_admin_types.py +++ b/tests/server/test_zones_admin_types.py @@ -243,25 +243,22 @@ async def test_remove_type_not_allowlisted_400(app, editor_override): # ── Integration with the real zones_db helpers (no SQL mocks) ───────────────── -def test_add_then_remove_roundtrip_via_real_helpers(tmp_path): +def test_add_then_remove_roundtrip_via_real_helpers(zones_schema): """Exercise the real add_zone_type / remove_zone_type helpers against a - throwaway zones.db. Catches schema drift or SQL typos that the + leased scratch zones schema. Catches schema drift or SQL typos that the route-level mock tests above would miss. No HTTP layer — just the - SQLite helpers — so we sidestep the run_sync ↔ mock-recursion issue + storage helpers — so we sidestep the run_sync ↔ mock-recursion issue and keep this test focused on the storage contract.""" from backend.eq2db import zones as zones_db + from tests.fixtures.pg import pg_conn - db_path = tmp_path / "zones.db" - conn = zones_db.ZoneCatalogue(db_path).init_db() - try: + db_path = zones_schema + with pg_conn(zones_schema) as conn: conn.execute( "INSERT INTO zones (id, name, name_lower, expansion_short, expansion_name, " "expansion_confidence) VALUES (1, 'Crushbone Keep', 'crushbone keep', 'RoK', " "'Rise of Kunark', 'category')" ) - conn.commit() - finally: - conn.close() # Add r1 = zones_db.ZoneCatalogue(db_path).add_zone_type("Crushbone Keep", "dungeon") From 3aedd2c39937143099bc47465fe6ca448caebfa5 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Wed, 7 Oct 2026 13:35:01 +0100 Subject: [PATCH 27/32] =?UTF-8?q?fix(pg):=20copy=20script=20loads=20zones?= =?UTF-8?q?=20before=20parses=20=E2=80=94=20tier=20classification=20needs?= =?UTF-8?q?=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detail-retention tiers classify each encounter's zone via _classify_zone, which reads the zones schema; copying parses first would classify every kill as "other" (7-day tier) against stale/empty zones data and over-prune curated raid detail at load. Also gitignore /restore/ and /reports/ (restored prod replicas + orphan quarantines — real user data, never committed). Co-Authored-By: Claude Fable 5 --- .gitignore | 5 +++++ scripts/migrate_to_postgres.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 92dae83b..849211d8 100644 --- a/.gitignore +++ b/.gitignore @@ -350,3 +350,8 @@ scripts/dev/.epics_wikitext_cache.json # Local MCP server config (Claude Code) — carries the Supabase personal # access token. Machine-local only, never committed. .mcp.json + +# Cutover artifacts: restored prod SQLite replicas + copy-script audit +# reports (both carry real user data) — never committed. +/restore/ +/reports/ diff --git a/scripts/migrate_to_postgres.py b/scripts/migrate_to_postgres.py index ba519578..4fb42917 100644 --- a/scripts/migrate_to_postgres.py +++ b/scripts/migrate_to_postgres.py @@ -63,8 +63,12 @@ REPORTS_DIR = _REPO / "reports" -#: family schema → (cli flag name). Copy order matters only within a family. -FAMILIES = ("users", "parses", "census", "zones", "raids") +#: Copy order. Cross-family FKs don't exist, but ZONES MUST PRECEDE PARSES: +#: the parses detail-retention tiers classify each encounter's zone via +#: _classify_zone, which reads the zones schema — with stale/empty zones +#: data every kill would classify "other" (7-day tier) and curated raid +#: detail would be over-pruned at load. +FAMILIES = ("users", "zones", "raids", "census", "parses") #: Tables whose rows are intentionally NOT copied (none today; placeholder #: so a future exclusion is one line). From 13640c986d90e94854412d9a6283761415231cf5 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Wed, 7 Oct 2026 14:04:53 +0100 Subject: [PATCH 28/32] =?UTF-8?q?perf(pg):=20cached=20zone=20canonical-nam?= =?UTF-8?q?e=20map=20=E2=80=94=20Wuoshi=20rankings=20rebuild=20276s=20->?= =?UTF-8?q?=200.9s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rehearsal #1 against real prod data (15k winning encounters) exposed a per-encounter zones_db.find_by_name round trip in _resolve_boss (and the same latent pattern in _classify_zone's miss path) — invisible under in-process SQLite, 18ms each on Postgres. One query now builds a process-cached {name_lower/alias_lower: canonical} map (ZoneCatalogue.canonical_name_map), consumed by both sites and cleared by invalidate_zones_cache with the other zone caches. End-to-end Wuoshi rebuild on the real 6.8GB dataset: 10,212 kills in 0.9s. Tests re-point their fakes at the new seam. Co-Authored-By: Claude Fable 5 --- backend/eq2db/zones.py | 12 ++++++++++++ backend/eq2db/zones.sql | 9 +++++++++ backend/server/api/parses/list.py | 16 +++++++++++++--- backend/server/api/rankings.py | 19 +++++++++++++++---- tests/server/test_parses_classify_zone.py | 10 +++++----- tests/server/test_rankings.py | 7 ++++--- 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/backend/eq2db/zones.py b/backend/eq2db/zones.py index a0857986..bf7d333b 100644 --- a/backend/eq2db/zones.py +++ b/backend/eq2db/zones.py @@ -1279,6 +1279,18 @@ def find_by_name(self, name: str) -> dict | None: z = Zone.find_by_name(name, schema=self.schema) return z.to_dict() if z is not None else None + def canonical_name_map(self) -> dict[str, str]: + """{lookup_key_lower: canonical zone name} for every canonical name + AND alias — one query, built for the process-level resolver caches + (per-encounter ``find_by_name`` calls in the rankings rebuild cost + a Postgres round trip each; 15k of them turned the rebuild into + minutes).""" + conn = _connect(self.schema) + try: + return {r["key"]: r["name"] for r in conn.execute(_SQL["list_zone_name_keys"]).fetchall()} + finally: + conn.close() + def list_by_expansion(self, short: str, type_filter: str | None = None) -> list[dict]: """All zones in an expansion as legacy hydrated dicts. ``type_filter`` stays positional for the existing call sites; the model exposes it as diff --git a/backend/eq2db/zones.sql b/backend/eq2db/zones.sql index d6563a8a..39bb2520 100644 --- a/backend/eq2db/zones.sql +++ b/backend/eq2db/zones.sql @@ -422,3 +422,12 @@ WHERE category = %s -- :name delete_featured_raid_category DELETE FROM featured_raid_categories WHERE expansion_short = %s AND name = %s; + +-- Every lookup key (canonical name_lower + every alias_lower) with its +-- canonical zone name — one pass for the process-cached resolver maps +-- (rankings._zone_canonical_map / parses list's classifier). Replaces +-- per-encounter find_by_name round trips in the rankings rebuild. +-- :name list_zone_name_keys +SELECT name_lower AS key, name FROM zones +UNION ALL +SELECT a.alias_lower AS key, z.name FROM zone_aliases a JOIN zones z ON z.id = a.zone_id; diff --git a/backend/server/api/parses/list.py b/backend/server/api/parses/list.py index b783b50d..82df7094 100644 --- a/backend/server/api/parses/list.py +++ b/backend/server/api/parses/list.py @@ -17,7 +17,6 @@ import psycopg from fastapi import HTTPException, Request -from backend.eq2db.zones import catalogue as zones_db from backend.server import db as users_db from backend.server.api.parses import router # the package-level router from backend.server.api.parses.models import ( @@ -198,6 +197,17 @@ def _cached_zones_data() -> tuple[dict, list[dict], list[dict], set[str]]: return _real() +def _zone_canonical_map() -> dict[str, str]: + """Thin local wrapper around rankings._zone_canonical_map — same + cycle-breaking indirection (and test seam) as _cached_zones_data. + One cached map instead of a per-miss ``zones_db.find_by_name`` round + trip: the cleanup sweep and list classification call _classify_zone + per encounter.""" + from backend.server.api.rankings import _zone_canonical_map as _real # noqa: PLC0415 + + return _real() + + _LEADERBOARD_MAP: dict[str, Literal["raid", "dungeon"]] | None = None @@ -244,9 +254,9 @@ def _classify_zone(zone: str | None) -> Literal["raid", "dungeon", "other"]: hit = _LEADERBOARD_MAP.get(zone.lower()) if hit is not None: return hit - canonical = zones_db.find_by_name(zone) + canonical = _zone_canonical_map().get(zone.lower()) if canonical: - hit = _LEADERBOARD_MAP.get(canonical["name"].lower()) + hit = _LEADERBOARD_MAP.get(canonical.lower()) if hit is not None: return hit return "other" diff --git a/backend/server/api/rankings.py b/backend/server/api/rankings.py index 87ef1a4e..8b5e7a45 100644 --- a/backend/server/api/rankings.py +++ b/backend/server/api/rankings.py @@ -318,6 +318,7 @@ def invalidate_zones_cache() -> None: parses_db.invalidate_is_player_cache() _encounter_required_mobs.cache_clear() _raid_boss_names.cache_clear() + _zone_canonical_map.cache_clear() @lru_cache(maxsize=1) @@ -412,6 +413,16 @@ def _tree_for_type(type_token: str) -> list[dict]: conn.close() +@lru_cache(maxsize=1) +def _zone_canonical_map() -> dict[str, str]: + """{name_lower-or-alias_lower: canonical zone name}, one query, cached + per process. _resolve_boss runs once per winning encounter — a + per-call ``zones_db.find_by_name`` round trip put 15k queries on the + Wuoshi rebuild path (276s measured at rehearsal). Cleared by + invalidate_zones_cache().""" + return zones_db.canonical_name_map() + + @lru_cache(maxsize=1) def _encounter_required_mobs() -> dict[tuple[str, str], frozenset[str]]: """(canonical zone, encounter name) → normalised mob keys, for curated @@ -477,14 +488,14 @@ def _resolve_boss(title: str, zone: str | None, scope: str) -> tuple[bool, str | candidates = boss_index.get(_normalise_boss_key(title)) if candidates: if len(candidates) > 1 and zone: - resolved = zones_db.find_by_name(zone) - if resolved: + canonical = _zone_canonical_map().get(zone.lower()) + if canonical: for cz, ct in candidates: - if cz == resolved["name"]: + if cz == canonical: return True, cz, ct cz, ct = candidates[0] return True, cz, ct - if zone and (resolved := zones_db.find_by_name(zone)) and resolved["name"] in curated_zones: + if zone and _zone_canonical_map().get(zone.lower()) in curated_zones: return False, zone, title if is_boss(title): return True, zone, title diff --git a/tests/server/test_parses_classify_zone.py b/tests/server/test_parses_classify_zone.py index 6c6554d1..9f469272 100644 --- a/tests/server/test_parses_classify_zone.py +++ b/tests/server/test_parses_classify_zone.py @@ -96,7 +96,7 @@ def test_classify_is_case_insensitive(): def test_classify_resolves_aliases(): # When the parse's `zone` doesn't match a canonical name directly but - # zones_db.find_by_name resolves it to one that's on the leaderboard, + # the canonical-name map resolves it to one that's on the leaderboard, # the classifier should still bucket it correctly. with ( patch( @@ -104,8 +104,8 @@ def test_classify_resolves_aliases(): return_value=_fake_trees(["Castle Mistmoore"], []), ), patch( - "backend.server.api.parses.list.zones_db.find_by_name", - return_value={"name": "Castle Mistmoore"}, + "backend.server.api.parses.list._zone_canonical_map", + return_value={"mistmoore castle": "Castle Mistmoore"}, ), ): _classifier_cache_clear() @@ -119,8 +119,8 @@ def test_classify_falls_through_to_other_when_alias_misses(): return_value=_fake_trees(["Castle Mistmoore"], []), ), patch( - "backend.server.api.parses.list.zones_db.find_by_name", - return_value=None, + "backend.server.api.parses.list._zone_canonical_map", + return_value={}, ), ): _classifier_cache_clear() diff --git a/tests/server/test_rankings.py b/tests/server/test_rankings.py index f7f6195a..25cada84 100644 --- a/tests/server/test_rankings.py +++ b/tests/server/test_rankings.py @@ -870,10 +870,11 @@ def test_resolve_boss_curated_zone_rejects_unmatched_titles(): "backend.server.api.rankings._cached_zones_data", return_value=(fake_boss_index, [], [], {"Throne of New Tunaria"}), ), - patch("backend.server.api.rankings.zones_db") as zdb, + patch( + "backend.server.api.rankings._zone_canonical_map", + return_value={"throne of new tunaria": "Throne of New Tunaria"}, + ), ): - zdb.find_by_name.side_effect = lambda z: {"name": z} if z == "Throne of New Tunaria" else None - # A player kill shaped like a named: rejected in the curated zone. ok, _, _ = _resolve_boss("Ripclaw", "Throne of New Tunaria", "raid") assert ok is False From 8e292e35358b61f52d9ad25a141ea749a0590e35 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Wed, 7 Oct 2026 14:18:03 +0100 Subject: [PATCH 29/32] =?UTF-8?q?perf(pg):=20rankings=20kills=20cache=20tt?= =?UTF-8?q?l=2060s=20->=2010min=20=E2=80=94=20Supabase=20egress=20budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each rebuild reads ~40 MB of combatants from Postgres (measured on prod data); a 60s SWR ttl under constant traffic could exceed Pro's 250 GB/month egress. At 10 min: worst-case ~175 GB, realistic 30-60 GB incl. nightly pg_dump. SWR semantics unchanged — boards still serve instantly; fresh uploads appear within <=10 min (user-approved trade). Co-Authored-By: Claude Fable 5 --- backend/server/api/rankings.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/backend/server/api/rankings.py b/backend/server/api/rankings.py index 8b5e7a45..a67ac322 100644 --- a/backend/server/api/rankings.py +++ b/backend/server/api/rankings.py @@ -99,11 +99,15 @@ def _normalise_boss_key(s: str) -> str: _METRIC_FIELD = {"dps": "encdps", "hps": "enchps"} # speed handled separately # Short-lived cache of the expensive load+group step (boards are cheap on top). -# ttl=60 keeps boards fresh-ish; max_age=6h means a stale board SERVES -# instantly (with a background rebuild) rather than blocking a visitor on -# the full parses scan — only a completely cold cache (deploy, 6h idle) -# builds inline. -rankings_cache: TTLCache = TTLCache(ttl=60, max_age=6 * 3600, name="rankings", maxsize=4) +# ttl=600 (user decision 2026-10-07): each rebuild reads every winning +# kill's combatants from Postgres — ~40 MB of Supabase egress per rebuild +# on 2026-10 data — so a 60s ttl under constant traffic could blow the Pro +# plan's 250 GB/month; at 10 min the worst case is ~175 GB and realistic +# usage ~30-60 GB. Fresh uploads reach the boards within ≤10 min. +# max_age=6h means a stale board SERVES instantly (with a background +# rebuild) rather than blocking a visitor on the full parses scan — only +# a completely cold cache (deploy, 6h idle) builds inline. +rankings_cache: TTLCache = TTLCache(ttl=600, max_age=6 * 3600, name="rankings", maxsize=4) _KILLS_KEY = "primary_boss_kills" From 1da5772ae6f3bd1edafcf9e708ada4727325a6ed Mon Sep 17 00:00:00 2001 From: VortexUK Date: Wed, 7 Oct 2026 14:27:27 +0100 Subject: [PATCH 30/32] fix(pg): fail fast when pools open on Windows' ProactorEventLoop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plain `uvicorn backend.server.app:app` on Windows (uvicorn 0.52 picks Proactor without --reload) left the async pool unable to create any connection — every checkout timed out after 30s and the site rendered as a blank page. open_pools now raises a crisp error naming the three working entry points instead. Production Linux unaffected. Co-Authored-By: Claude Fable 5 --- backend/pg.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/pg.py b/backend/pg.py index 7e6d052e..f088de40 100644 --- a/backend/pg.py +++ b/backend/pg.py @@ -61,6 +61,20 @@ async def open_pools() -> None: Small on purpose: WEB_CONCURRENCY=1 and web+bot share one process, so ~10 server connections total stays well inside Supabase's session- pooler limits.""" + import asyncio + import sys + + if sys.platform == "win32" and isinstance(asyncio.get_running_loop(), asyncio.ProactorEventLoop): + # Fail fast and loud: psycopg's async side can't create connections + # on the ProactorEventLoop, and the symptom otherwise is every + # async checkout silently timing out after 30s (a blank site). + # Plain `uvicorn backend.server.app:app` lands here on Windows + # (uvicorn 0.52 picks Proactor without --reload). + raise RuntimeError( + "psycopg async cannot run on Windows' ProactorEventLoop. Start the app " + "via `python main.py`, or uvicorn WITH --reload (scripts/dev_backend.ps1), " + "or call pg.ensure_selector_event_loop_policy() before the loop is created." + ) global _async_pool, _sync_pool _async_pool = AsyncConnectionPool(dsn(), min_size=1, max_size=5, open=False, kwargs={"row_factory": dict_row}) await _async_pool.open() From 1f8833e6ff753466fee78e9d8115f4ec8d72adc1 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Wed, 7 Oct 2026 14:40:06 +0100 Subject: [PATCH 31/32] feat(pg): ANALYZE each table after the cutover load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh planner statistics at commit time — the first post-cutover queries (the rankings rebuild over millions of just-loaded rows) must not plan against empty stats while autovacuum catches up. (supabase-postgres-best-practices: monitor-vacuum-analyze.) Co-Authored-By: Claude Fable 5 --- scripts/migrate_to_postgres.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/migrate_to_postgres.py b/scripts/migrate_to_postgres.py index 4fb42917..0cfb97ed 100644 --- a/scripts/migrate_to_postgres.py +++ b/scripts/migrate_to_postgres.py @@ -340,6 +340,12 @@ def copy_family( (f"{schema}.{table}", col), ) + # Fresh planner statistics immediately — the first post-cutover queries + # (the rankings rebuild) must not run against empty stats while + # autovacuum catches up on millions of just-loaded rows. + for table in order: + conn.execute(f"ANALYZE {table}") + # Verify counts inside the same transaction. for table in order: key = f"{schema}.{table}" From aa56ba785d8b5bee0dcfdd238475670ae218a954 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Wed, 7 Oct 2026 14:43:05 +0100 Subject: [PATCH 32/32] =?UTF-8?q?fix(pg):=20copy=20script=20commits=20per?= =?UTF-8?q?=20table=20=E2=80=94=20bounded=20WAL=20for=20bulk=20loads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single family-wide transaction accumulated WAL across the whole 3.25M-row parses load and filled a freshly-upgraded Supabase project's small provisioned disk mid-COPY (DiskFull at attack_types row 1.34M). Per-table commits let checkpoints recycle WAL between tables; idempotency is unchanged (every run starts by truncating the family, and within-family FK order keeps committed parents valid). Co-Authored-By: Claude Fable 5 --- scripts/migrate_to_postgres.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/migrate_to_postgres.py b/scripts/migrate_to_postgres.py index 0cfb97ed..d6bb5d94 100644 --- a/scripts/migrate_to_postgres.py +++ b/scripts/migrate_to_postgres.py @@ -13,8 +13,9 @@ POSTGRES_CONNECTION_STRING) or --dsn. Safety: the script prints the target host and requires --yes (or interactive confirmation) before writing. -Per family (one transaction each): TRUNCATE … RESTART IDENTITY CASCADE → -COPY each table in FK order → setval every identity sequence → COMMIT. +Per family: TRUNCATE … RESTART IDENTITY CASCADE → COPY each table in FK +order (committed PER TABLE — bounds WAL so a bulk load can't fill a small +disk) → setval every identity sequence → ANALYZE → final verify. Column lists and types are introspected from information_schema, so the copy follows the reviewed PG schemas; transforms applied by type: @@ -331,6 +332,13 @@ def copy_family( (now, list(pruned_encounters)), ) + # Commit per table: bounds the WAL a bulk load accumulates (a single + # family-wide transaction filled a small Supabase disk mid-COPY) and + # lets checkpoints recycle between tables. Idempotency is unchanged — + # every run starts by truncating the family, and the within-family FK + # order means a committed parent is always valid for its children. + conn.commit() + # setval every identity sequence to MAX(col). for table in order: for col in pg_identity_columns(conn, schema, table): @@ -346,7 +354,7 @@ def copy_family( for table in order: conn.execute(f"ANALYZE {table}") - # Verify counts inside the same transaction. + # Verify counts (tables were committed individually above). for table in order: key = f"{schema}.{table}" trep = report.table(key)