diff --git a/scripts/fastapi/cache_dumper.py b/scripts/fastapi/cache_dumper.py index 92e70e4..e8012bf 100644 --- a/scripts/fastapi/cache_dumper.py +++ b/scripts/fastapi/cache_dumper.py @@ -28,6 +28,40 @@ def chunks(lst: List, n: int): yield lst[i:i + n] +def convert_value_to_dynamodb(value): + """ + Convert a single Python value to DynamoDB format (recursive helper) + + Args: + value: Python value of any type + + Returns: + DynamoDB formatted value with type key + """ + if isinstance(value, bool): + # IMPORTANT: Check bool before int (bool is subclass of int in Python) + return {'BOOL': value} + elif isinstance(value, str): + return {'S': value} + elif isinstance(value, int): + return {'N': str(value)} + elif isinstance(value, float): + return {'N': str(value)} + elif isinstance(value, dict): + return {'M': convert_to_dynamodb_format(value)} + elif isinstance(value, list): + # Recursively convert each item in the list + dynamodb_list = [] + for item in value: + dynamodb_list.append(convert_value_to_dynamodb(item)) + return {'L': dynamodb_list} + elif value is None: + return {'NULL': True} + else: + # Fallback: convert to string + return {'S': str(value)} + + def convert_to_dynamodb_format(data: Dict) -> Dict: """ Convert Python dict to DynamoDB format @@ -41,29 +75,7 @@ def convert_to_dynamodb_format(data: Dict) -> Dict: dynamodb_item = {} for key, value in data.items(): - if isinstance(value, str): - dynamodb_item[key] = {'S': value} - elif isinstance(value, int): - dynamodb_item[key] = {'N': str(value)} - elif isinstance(value, float): - dynamodb_item[key] = {'N': str(value)} - elif isinstance(value, bool): - dynamodb_item[key] = {'BOOL': value} - elif isinstance(value, dict): - dynamodb_item[key] = {'M': convert_to_dynamodb_format(value)} - elif isinstance(value, list): - # Convert list to DynamoDB List - dynamodb_list = [] - for item in value: - if isinstance(item, str): - dynamodb_list.append({'S': item}) - elif isinstance(item, (int, float)): - dynamodb_list.append({'N': str(item)}) - elif isinstance(item, dict): - dynamodb_list.append({'M': convert_to_dynamodb_format(item)}) - dynamodb_item[key] = {'L': dynamodb_list} - elif value is None: - dynamodb_item[key] = {'NULL': True} + dynamodb_item[key] = convert_value_to_dynamodb(value) return dynamodb_item @@ -127,104 +139,204 @@ def batch_write_to_dynamodb(table_name: str, items: List[Dict]) -> Dict: async def dump_cache_to_db() -> Dict: """ - Dump all dirty cache entries to DynamoDB + Dump all dirty cache entries to DynamoDB by replaying WAL operations. + + CRITICAL FIX: Instead of dumping raw cache entries (which have wrapped structures + like {"success": True, "data": [...]}), we now read from the WAL operation log + which has correctly structured operations for DynamoDB updates. Returns: Dictionary with dump statistics """ - info("๐Ÿ—„๏ธ Starting cache dump to DynamoDB...") + info("๐Ÿ—„๏ธ Starting cache dump to DynamoDB via WAL operations...") try: - # Get all dirty entries from cache - dirty_entries = cache_manager.get_dirty_entries() - - if not dirty_entries: - info("โœ… No dirty entries to dump") - return {"success": True, "entries": 0, "message": "No dirty entries"} - - info(f"๐Ÿ“ฆ Found {len(dirty_entries)} dirty entries to dump") - - # Group entries by table - users_items = [] - daily_items = [] - duels_items = [] - bounties_items = [] - - for entry in dirty_entries: - cache_type = entry.get('cache_type') + # Get checkpoint to avoid replaying already-applied entries + last_applied = wal_manager.get_last_applied_sequence() + info(f"๐Ÿ“ Checkpoint: last_applied_sequence = {last_applied}") + + # Get WAL entries since last checkpoint (+ 1 to get next unapplied entry) + wal_entries = wal_manager.get_entries_since(last_applied + 1) + + if not wal_entries: + info("โœ… No new WAL entries to sync") + return {"success": True, "entries": 0, "message": "No new WAL entries"} + + info(f"๐Ÿ“ฆ Found {len(wal_entries)} new WAL entries to sync to DynamoDB") + + # Track stats + total_synced = 0 + total_failed = 0 + errors = [] + + # Process each WAL operation until first failure + # CRITICAL: We must stop on first failure to prevent double-applying later INCREMENTs + # Example: seq 1 โœ…, seq 2 โŒ, seq 3 INCREMENT โœ… (checkpoint=1) + # Next run: seq 2 fails again, seq 3 replayed โ†’ INCREMENT applied twice! + for entry in wal_entries: + operation = entry.get('operation') + table = entry.get('table') + key = entry.get('key') data = entry.get('data') - if not data: - continue - - # Convert to DynamoDB format - dynamodb_item = convert_to_dynamodb_format(data) + # Track sequence for checkpoint management + entry_sequence = entry.get('sequence', -1) + + # Validate based on operation type + # DELETE operations don't require data, all others do + if operation == "DELETE": + if not all([operation, table, key]): + total_failed += 1 + error_msg = f"Incomplete DELETE entry at sequence {entry_sequence}: {entry}" + warning(error_msg) + errors.append(error_msg) + # STOP processing to prevent replaying later entries (especially INCREMENTs) + break + else: + if not all([operation, table, key, data]): + total_failed += 1 + error_msg = f"Incomplete {operation} entry at sequence {entry_sequence}: {entry}" + warning(error_msg) + errors.append(error_msg) + # STOP processing to prevent replaying later entries (especially INCREMENTs) + break - # CRITICAL FIX: Convert string cache_type to CacheType enum for comparison - # cache_type is a string (e.g., "users") from cache key, not a CacheType enum try: - cache_type_enum = CacheType(cache_type) - except ValueError: - error(f"Unknown cache type: {cache_type}, skipping entry") - continue - - # Route to appropriate table - if cache_type_enum == CacheType.USERS: - users_items.append(dynamodb_item) - elif cache_type_enum == CacheType.DAILY_PROBLEM or cache_type_enum == CacheType.DAILY_COMPLETIONS: - daily_items.append(dynamodb_item) - elif cache_type_enum == CacheType.DUELS: - duels_items.append(dynamodb_item) - elif cache_type_enum == CacheType.BOUNTIES or cache_type_enum == CacheType.BOUNTY_COMPETITIONS: - bounties_items.append(dynamodb_item) - - # Batch write to each table - results = {} - - if users_items: - info(f"๐Ÿ’พ Writing {len(users_items)} users to DynamoDB...") - results['users'] = batch_write_to_dynamodb(USERS_TABLE, users_items) - - if daily_items: - info(f"๐Ÿ’พ Writing {len(daily_items)} daily items to DynamoDB...") - results['daily'] = batch_write_to_dynamodb(DAILY_TABLE, daily_items) - - if duels_items: - info(f"๐Ÿ’พ Writing {len(duels_items)} duels to DynamoDB...") - results['duels'] = batch_write_to_dynamodb(DUELS_TABLE, duels_items) - - if bounties_items: - info(f"๐Ÿ’พ Writing {len(bounties_items)} bounties to DynamoDB...") - results['bounties'] = batch_write_to_dynamodb(BOUNTIES_TABLE, bounties_items) - - # Check overall success - all_successful = all(r.get('success', False) for r in results.values()) - - if all_successful: - # Mark all entries as synced - for entry in dirty_entries: - cache_manager.mark_synced(entry['cache_type'], entry.get('identifier', '')) - - # Clear WAL file - wal_manager.clear() - - total_written = sum(r.get('written', 0) for r in results.values()) - info(f"โœ… Cache dump complete: {total_written} entries written to DynamoDB") + if operation == "UPDATE": + # Build UpdateExpression from data + update_expr_parts = [] + expr_attr_values = {} + + for field, value in data.items(): + update_expr_parts.append(f"{field} = :{field}") + # Convert to DynamoDB format using helper to preserve types + expr_attr_values[f":{field}"] = convert_value_to_dynamodb(value) + + # Convert key to DynamoDB format + dynamodb_key = {} + for k, v in key.items(): + if isinstance(v, str): + dynamodb_key[k] = {'S': v} + elif isinstance(v, (int, float)): + dynamodb_key[k] = {'N': str(v)} + + # Perform update + ddb.update_item( + TableName=table, + Key=dynamodb_key, + UpdateExpression=f"SET {', '.join(update_expr_parts)}", + ExpressionAttributeValues=expr_attr_values + ) + + elif operation == "PUT": + # Full item put - merge key and data + item = {**key, **data} + dynamodb_item = convert_to_dynamodb_format(item) + + ddb.put_item( + TableName=table, + Item=dynamodb_item + ) + + elif operation == "DELETE": + # Convert key to DynamoDB format + dynamodb_key = {} + for k, v in key.items(): + if isinstance(v, str): + dynamodb_key[k] = {'S': v} + elif isinstance(v, (int, float)): + dynamodb_key[k] = {'N': str(v)} + + ddb.delete_item( + TableName=table, + Key=dynamodb_key + ) + + elif operation == "INCREMENT": + # Build increment expression + update_expr_parts = [] + expr_attr_values = {} + + for field, value in data.items(): + update_expr_parts.append(f"{field} = if_not_exists({field}, :zero) + :{field}") + expr_attr_values[f":{field}"] = {'N': str(value)} + + expr_attr_values[":zero"] = {'N': '0'} + + # Convert key to DynamoDB format + dynamodb_key = {} + for k, v in key.items(): + if isinstance(v, str): + dynamodb_key[k] = {'S': v} + elif isinstance(v, (int, float)): + dynamodb_key[k] = {'N': str(v)} + + ddb.update_item( + TableName=table, + Key=dynamodb_key, + UpdateExpression=f"SET {', '.join(update_expr_parts)}", + ExpressionAttributeValues=expr_attr_values + ) + + else: + # Unknown operation type - fail explicitly + total_failed += 1 + error_msg = f"Unknown WAL operation type '{operation}' at sequence {entry_sequence}" + warning(error_msg) + errors.append(error_msg) + # STOP processing to prevent replaying later entries (especially INCREMENTs) + break + + total_synced += 1 + + # Update checkpoint after each successful operation + # This is safe because we STOP on first failure (break above) + if entry_sequence >= 0: + wal_manager.set_last_applied_sequence(entry_sequence) + + except Exception as e: + total_failed += 1 + error_msg = f"Failed to sync WAL entry at sequence {entry_sequence} to {table}: {e}" + error(error_msg) + errors.append(error_msg) + # STOP processing to prevent replaying later entries (especially INCREMENTs) + break + + # Mark success if all synced + if total_failed == 0: + # CRITICAL FIX: Clear WAL entries ONLY up to the last successfully applied sequence + # This prevents race condition where new writes after our snapshot are lost + + # Get the highest sequence we successfully applied + last_synced_sequence = wal_manager.get_last_applied_sequence() + + # Clear WAL entries up to last_synced_sequence, keeping any concurrent writes + # that occurred after our snapshot (sequence > last_synced_sequence) + wal_manager.clear_up_to(last_synced_sequence) + + # Note: We intentionally DON'T call cache_manager.mark_synced() here because: + # - Cache entries don't track their WAL sequence number + # - We can't safely determine which cache entries map to synced WAL entries + # - Dirty flags in cache are eventually consistent (background task marks them synced) + # - The WAL checkpoint is our source of truth for what's been persisted + + info(f"โœ… Cache dump complete: {total_synced} WAL operations synced to DynamoDB (checkpoint: {last_synced_sequence})") return { "success": True, - "entries": total_written, - "results": results + "entries": total_synced, + "failed": 0, + "checkpoint": last_synced_sequence } else: - total_failed = sum(r.get('failed', 0) for r in results.values()) - error(f"โš ๏ธ Cache dump partially failed: {total_failed} entries failed") + warning(f"โš ๏ธ Cache dump partially failed: {total_failed}/{len(wal_entries)} operations failed") return { "success": False, - "entries": len(dirty_entries), + "entries": len(wal_entries), + "synced": total_synced, "failed": total_failed, - "results": results + "errors": errors } except Exception as e: diff --git a/scripts/fastapi/routes/admin.py b/scripts/fastapi/routes/admin.py index d69903d..0416c11 100644 --- a/scripts/fastapi/routes/admin.py +++ b/scripts/fastapi/routes/admin.py @@ -153,3 +153,83 @@ async def get_log_content( return {"success": True, "content": content, "path": log_path} except Exception as error: return {"success": False, "error": str(error)} + + +@router.get("/cache/status") +async def get_cache_status( + api_key: str = Depends(verify_api_key_query) +): + """Get comprehensive cache status including all entries and WAL stats + + Access via: /admin/cache/status?api_key=YOUR_API_KEY + + Returns: + - Cache stats (size, hit rate, entries per type) + - WAL stats (entries, checkpoint, file size) + - Sample of cache entries (keys only, for privacy) + - Dirty entries count + """ + try: + from cache_manager import cache_manager + from wal_manager import wal_manager + + # Get cache stats + cache_stats = cache_manager.get_stats() + + # Get WAL stats + wal_stats = wal_manager.get_stats() + + # Get dirty entries info (without exposing data) + dirty_entries = cache_manager.get_dirty_entries() + dirty_summary = [] + for entry in dirty_entries: + dirty_summary.append({ + "cache_type": entry.get('cache_type'), + "identifier": entry.get('identifier', '(no identifier)'), + "timestamp": entry.get('timestamp'), + "last_synced": entry.get('last_synced') + }) + + # Get cache keys by type (for debugging) + # Note: Accessing _cache directly for admin debugging only + # TODO: Add public method to CacheManager for proper encapsulation + cache_keys_by_type = {} + for cache_type in ["users", "duels", "bounties", "daily_problem", "daily_completions", "user_daily_data"]: + keys = [k for k in cache_manager._cache.keys() if k.startswith(f"{cache_type}:")] + cache_keys_by_type[cache_type] = { + "count": len(keys), + "sample_keys": keys[:5] # Only show first 5 for privacy + } + + return { + "success": True, + "data": { + "cache": cache_stats, + "wal": wal_stats, + "dirty_entries": { + "count": len(dirty_entries), + "entries": dirty_summary[:10] # Only show first 10 + }, + "cache_keys_by_type": cache_keys_by_type, + "timestamp": datetime.utcnow().isoformat() + } + } + except Exception as error: + return {"success": False, "error": str(error)} + + +@router.post("/cache/dump") +async def trigger_cache_dump( + api_key: str = Depends(verify_api_key_query) +): + """Manually trigger cache dump to DynamoDB + + Access via: POST /admin/cache/dump?api_key=YOUR_API_KEY + """ + try: + from cache_dumper import dump_cache_to_db + result = await dump_cache_to_db() + # dump_cache_to_db already returns {"success": ..., ...}, don't double-wrap + return result + except Exception as error: + return {"success": False, "error": str(error)} diff --git a/scripts/fastapi/routes/duels.py b/scripts/fastapi/routes/duels.py index 501de75..93a0c19 100644 --- a/scripts/fastapi/routes/duels.py +++ b/scripts/fastapi/routes/duels.py @@ -190,7 +190,8 @@ async def get_duel_endpoint( cached_duels = cache_manager.get(CacheType.DUELS) if cached_duels: for duel in cached_duels.get('data', []): - if duel.get('id') == duel_id: + # FIXED: Use 'duelId' not 'id' to match DynamoDB schema + if duel.get('duelId') == duel_id: return {"success": True, "data": duel} # Fallback to database diff --git a/scripts/fastapi/static/log_viewer.html b/scripts/fastapi/static/log_viewer.html index 7c9450f..9118d7c 100644 --- a/scripts/fastapi/static/log_viewer.html +++ b/scripts/fastapi/static/log_viewer.html @@ -320,6 +320,130 @@ .filter-info strong { color: #4ec9b0; } + + .tab-button { + background: #3e3e42; + color: #d4d4d4; + border: none; + padding: 12px 20px; + border-radius: 6px; + cursor: pointer; + font-family: inherit; + font-size: 14px; + font-weight: 600; + transition: all 0.2s; + } + + .tab-button:hover { + background: #4e4e52; + } + + .tab-button.active { + background: #0e639c; + color: white; + } + + #cacheViewer { + display: none; + } + + #cacheViewer.active { + display: block; + } + + #logViewer.active { + display: block; + } + + #logViewer { + display: none; + } + + .cache-section { + background: #252526; + padding: 20px; + border-radius: 8px; + margin-bottom: 20px; + border: 1px solid #3e3e42; + } + + .cache-section h2 { + color: #4ec9b0; + font-size: 18px; + margin-bottom: 15px; + display: flex; + align-items: center; + gap: 10px; + } + + .cache-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 15px; + } + + .cache-item { + background: #1e1e1e; + padding: 15px; + border-radius: 6px; + border: 1px solid #3e3e42; + } + + .cache-item-label { + color: #858585; + font-size: 11px; + text-transform: uppercase; + margin-bottom: 8px; + } + + .cache-item-value { + color: #d4d4d4; + font-size: 16px; + font-weight: 600; + } + + .cache-item-value.highlight { + color: #4ec9b0; + font-size: 24px; + } + + .cache-keys-list { + max-height: 300px; + overflow-y: auto; + background: #1e1e1e; + padding: 10px; + border-radius: 4px; + border: 1px solid #3e3e42; + } + + .cache-key { + padding: 6px 10px; + margin-bottom: 4px; + background: #2d2d30; + border-radius: 3px; + font-size: 12px; + font-family: 'SF Mono', monospace; + color: #9cdcfe; + } + + .dirty-entry { + background: #3e2723; + border-left: 3px solid #f48771; + padding: 10px; + margin-bottom: 8px; + border-radius: 4px; + } + + .dirty-entry-header { + color: #f48771; + font-weight: 600; + margin-bottom: 5px; + } + + .dirty-entry-info { + color: #858585; + font-size: 11px; + } @@ -327,7 +451,12 @@

๐Ÿ“Š YeetCode FastAPI Log Viewer

-
+
+ + +
+ +
@@ -410,6 +539,8 @@

๐Ÿ“Š YeetCode FastAPI Log Viewer

+ +
+
+ + + +
+
+

๐Ÿ“Š Cache Statistics

+
+
+
Loading...
+
โณ
+
+
+
+ +
+

๐Ÿ—„๏ธ WAL Status

+
+
+
Loading...
+
โณ
+
+
+
+ +
+

โš ๏ธ Dirty Entries

+
+

Loading...

+
+
+ +
+

๐Ÿ”‘ Cache Keys by Type

+
+

Loading...

+
+
+ +
+ + +
+
+ diff --git a/scripts/fastapi/wal_manager.py b/scripts/fastapi/wal_manager.py index 8731a51..2c0021b 100644 --- a/scripts/fastapi/wal_manager.py +++ b/scripts/fastapi/wal_manager.py @@ -43,14 +43,17 @@ def __init__(self, wal_file_path: str = None): wal_file_path = str(wal_dir / "wal.log") self._wal_file = wal_file_path + self._checkpoint_file = wal_file_path.replace('.log', '.checkpoint') self._lock = threading.RLock() self._sequence = 0 + self._last_applied_sequence = -1 self._file_handle = None - # Initialize WAL file + # Initialize WAL file and checkpoint self._init_wal_file() + self._load_checkpoint() - info(f"๐Ÿ“ WAL Manager initialized: {self._wal_file}") + info(f"๐Ÿ“ WAL Manager initialized: {self._wal_file} (checkpoint: {self._last_applied_sequence})") def _init_wal_file(self): """Initialize WAL file if it doesn't exist""" @@ -173,6 +176,9 @@ def clear(self) -> bool: """ Clear the WAL file after successful cache dump + WARNING: This resets sequence to 0, breaking checkpoint-based replay! + Use clear_up_to(sequence) instead for production code. + Returns: True if successful, False otherwise """ @@ -190,6 +196,50 @@ def clear(self) -> bool: error(f"Failed to clear WAL: {e}") return False + def clear_up_to(self, max_sequence: int) -> bool: + """ + Clear WAL entries up to and including max_sequence, keeping later entries + + This is the correct way to clear WAL after partial sync without breaking + the checkpoint system or losing concurrent writes. + + Args: + max_sequence: Clear all entries with sequence <= this value + + Returns: + True if successful, False otherwise + """ + with self._lock: + try: + # Read all entries + entries_to_keep = [] + with open(self._wal_file, 'r') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + # Keep entries with sequence > max_sequence + if entry.get('sequence', 0) > max_sequence: + entries_to_keep.append(entry) + except json.JSONDecodeError: + continue + + # Rewrite WAL file with only entries to keep + with open(self._wal_file, 'w') as f: + for entry in entries_to_keep: + f.write(json.dumps(entry) + '\n') + f.flush() + os.fsync(f.fileno()) + + info(f"๐Ÿงน WAL file cleared up to sequence {max_sequence}, kept {len(entries_to_keep)} entries") + return True + + except Exception as e: + error(f"Failed to clear WAL up to sequence {max_sequence}: {e}") + return False + def get_entries_since(self, sequence: int) -> List[Dict]: """ Get all WAL entries since a specific sequence number @@ -222,6 +272,53 @@ def get_entries_since(self, sequence: int) -> List[Dict]: error(f"Failed to get WAL entries: {e}") return [] + def _load_checkpoint(self) -> None: + """Load last applied sequence from checkpoint file""" + try: + if os.path.exists(self._checkpoint_file): + with open(self._checkpoint_file, 'r') as f: + data = json.load(f) + self._last_applied_sequence = data.get('last_applied_sequence', -1) + info(f"๐Ÿ“ Loaded checkpoint: last_applied_sequence = {self._last_applied_sequence}") + else: + info("๐Ÿ“ No checkpoint file found, starting from sequence -1") + except Exception as e: + error(f"Failed to load checkpoint: {e}, starting from -1") + self._last_applied_sequence = -1 + + def get_last_applied_sequence(self) -> int: + """Get the sequence number of the last successfully applied WAL entry""" + with self._lock: + return self._last_applied_sequence + + def set_last_applied_sequence(self, sequence: int) -> bool: + """ + Update the last applied sequence checkpoint (atomic write) + + Args: + sequence: Sequence number of last successfully applied entry + + Returns: + True if successful, False otherwise + """ + with self._lock: + try: + # Write to temp file first, then atomic rename + temp_file = self._checkpoint_file + '.tmp' + with open(temp_file, 'w') as f: + json.dump({'last_applied_sequence': sequence}, f) + f.flush() + os.fsync(f.fileno()) + + # Atomic rename (overwrites existing checkpoint) + os.replace(temp_file, self._checkpoint_file) + + self._last_applied_sequence = sequence + return True + except Exception as e: + error(f"Failed to save checkpoint: {e}") + return False + def get_stats(self) -> Dict: """Get WAL statistics""" with self._lock: @@ -230,7 +327,8 @@ def get_stats(self) -> Dict: return { "exists": False, "entries": 0, - "size_bytes": 0 + "size_bytes": 0, + "last_applied_sequence": self._last_applied_sequence } size = os.path.getsize(self._wal_file) @@ -240,7 +338,8 @@ def get_stats(self) -> Dict: "entries": self._sequence, "size_bytes": size, "size_kb": round(size / 1024, 2), - "path": self._wal_file + "path": self._wal_file, + "last_applied_sequence": self._last_applied_sequence } except Exception as e: