From 9d3610e715573013f46c1a52ec70b7b6bb23b608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Brzezina?= Date: Sat, 10 Oct 2026 01:49:07 +0200 Subject: [PATCH 1/5] fix(provider-arr): accept 1-7 digit fractional seconds in NLog timestamps *arr apps write log files with the NLog layout yyyy-MM-dd HH:mm:ss.f (a single fractional digit), so real Sonarr/Radarr/Prowlarr/Whisparr lines did not match the parser regex, which required exactly three digits, and were dropped or treated as continuations. --- packages/provider-arr/src/arr.parser.spec.ts | 58 ++++++++++++++++++-- packages/provider-arr/src/arr.parser.ts | 6 +- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/packages/provider-arr/src/arr.parser.spec.ts b/packages/provider-arr/src/arr.parser.spec.ts index 3480f38..27da7f8 100644 --- a/packages/provider-arr/src/arr.parser.spec.ts +++ b/packages/provider-arr/src/arr.parser.spec.ts @@ -101,16 +101,66 @@ describe('Arr Parser', () => { expect(result?.raw).toBe(line); }); - it('should handle timestamp without milliseconds', () => { - const line = '2024-01-15 10:30:45|Info|Test|No milliseconds'; - // This should not match the NLog format since .123 is expected + it('should handle timestamp without fractional seconds', () => { + const line = '2024-01-15 10:30:45|Info|Test|No fractional seconds'; const result = parseArrLogLine(line); - // Actually checking the regex - it uses (?:\.\d{3})? which makes ms optional + expect(result).not.toBeNull(); + expect(result?.timestamp.getMilliseconds()).toBe(0); + expect(result?.message).toBe('No fractional seconds'); + }); + + describe('fractional second precision', () => { + it.each([ + ['.2', 200], + ['.25', 250], + ['.123', 123], + ['.1234', 123], + ['.1234567', 123], + ['', 0], + ])('should parse fractional seconds %j as %i ms', (fraction, expectedMs) => { + const line = `2026-10-10 12:00:01${fraction}|Info|RssSyncService|RSS Sync Completed. Reports found: 50`; + const result = parseArrLogLine(line); + + expect(result).not.toBeNull(); + expect(result?.level).toBe('info'); + expect(result?.source).toBe('RssSyncService'); + expect(result?.message).toBe('RSS Sync Completed. Reports found: 50'); + expect(result?.timestamp.getTime()).toBe( + new Date(2026, 9, 10, 12, 0, 1, expectedMs).getTime() + ); + }); }); }); describe('isArrLogContinuation', () => { + it('should not treat a one-digit fractional second line as a continuation', () => { + expect( + isArrLogContinuation('2026-10-10 12:00:01.2|Info|RssSyncService|RSS Sync Completed.') + ).toBe(false); + }); + + it('should parse a realistic multi-line exception sample', () => { + const lines = [ + '2026-10-10 12:00:05.4|Error|DownloadClientProvider|Error connecting to qBittorrent', + '', + '[v4.0.10.2544] System.Net.Http.HttpRequestException: Connection refused (localhost:8080)', + ' at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken cancellationToken)', + ]; + + const first = parseArrLogLine(lines[0] as string); + expect(first).not.toBeNull(); + expect(first?.level).toBe('error'); + expect(first?.source).toBe('DownloadClientProvider'); + expect(first?.timestamp.getTime()).toBe(new Date(2026, 9, 10, 12, 0, 5, 400).getTime()); + expect(isArrLogContinuation(lines[0] as string)).toBe(false); + + // Exception detail lines are not NLog entries and must be treated as continuations + expect(parseArrLogLine(lines[2] as string)).toBeNull(); + expect(isArrLogContinuation(lines[2] as string)).toBe(true); + expect(isArrLogContinuation(lines[3] as string)).toBe(true); + }); + it('should detect stack trace lines', () => { expect(isArrLogContinuation(' at System.Threading.Task.Run()')).toBe(true); expect(isArrLogContinuation(' at MyClass.MyMethod()')).toBe(true); diff --git a/packages/provider-arr/src/arr.parser.ts b/packages/provider-arr/src/arr.parser.ts index 5a860bb..1001c29 100644 --- a/packages/provider-arr/src/arr.parser.ts +++ b/packages/provider-arr/src/arr.parser.ts @@ -173,10 +173,12 @@ export const ARR_LOG_FILE_CONFIG: LogFileConfig = { /** * NLog timestamp + metadata regex - * Matches: 2024-01-15 10:30:45.123|Level|Component|Message + * Matches: 2024-01-15 10:30:45.1|Level|Component|Message + * The fractional second is optional and may have 1-7 digits: *arr apps log with + * the NLog layout `yyyy-MM-dd HH:mm:ss.f` (one digit), older/custom layouts use 3. */ const NLOG_LINE_REGEX = - /^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d{3})?)\|(\w+)\|([^|]+)\|(.*)$/; + /^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d{1,7})?)\|(\w+)\|([^|]+)\|(.*)$/; /** * Log level mapping from NLog format From 788ff03267ca8e60760b21c1cf255e3e5ec83e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Brzezina?= Date: Sat, 10 Oct 2026 02:55:19 +0200 Subject: [PATCH 2/5] fix(file-ingestion): emit *arr log file entries and attach exceptions LogFileProcessor finalizes the pending entry before asking the provider to parse a new header line, as the parseLogLineWithContext contract in core expects (Emby and others return the current entry). The *arr context parser instead returned the previous entry and stashed the current one in the caller's context, which the processor discards, so file-tailed Sonarr, Radarr, Prowlarr and Whisparr logs produced no entries at all. - parseArrLogLineWithContext now returns the current entry for header lines, flags continuations, and ignores blank lines (NLog writes one between a message and its exception) - Exception extraction moves to finalizeArrLogEntry, invoked through a new optional finalizeLogEntry hook in LogFileProcessor; it strips the "[vX.Y.Z.W]" prefix so the exception type and message are populated - End-to-end LogFileProcessor tests with real Sonarr and Emby providers --- .../file-ingestion/log-file-processor.spec.ts | 114 ++++++++++++++++ .../file-ingestion/log-file-processor.ts | 9 +- packages/provider-arr/src/arr.parser.spec.ts | 104 +++++++++++---- packages/provider-arr/src/arr.parser.ts | 122 +++++++----------- packages/provider-arr/src/arr.provider.ts | 9 ++ packages/provider-arr/src/index.ts | 1 + 6 files changed, 255 insertions(+), 104 deletions(-) create mode 100644 apps/backend/src/modules/file-ingestion/log-file-processor.spec.ts diff --git a/apps/backend/src/modules/file-ingestion/log-file-processor.spec.ts b/apps/backend/src/modules/file-ingestion/log-file-processor.spec.ts new file mode 100644 index 0000000..89ece45 --- /dev/null +++ b/apps/backend/src/modules/file-ingestion/log-file-processor.spec.ts @@ -0,0 +1,114 @@ +import { EmbyProvider } from '@logarr/provider-emby'; +import { SonarrProvider } from '@logarr/provider-sonarr'; +import { describe, expect, it } from 'vitest'; + +import { LogFileProcessor } from './log-file-processor'; + +import type { MediaServerProvider, ParsedLogEntry } from '@logarr/core'; + +function feed( + lines: string[], + provider: MediaServerProvider = new SonarrProvider() +): ParsedLogEntry[] { + const processor = new LogFileProcessor(provider); + const out: ParsedLogEntry[] = []; + lines.forEach((line, i) => { + const entry = processor.processLine(line, i + 1); + if (entry) out.push(entry); + }); + const last = processor.flush(); + if (last) out.push(last); + return out; +} + +describe('LogFileProcessor with *arr multi-line errors', () => { + it('assembles two entries and a blank-line separated exception with stack trace', () => { + const entries = feed([ + '2026-10-10 12:00:05.4|Error|DownloadClientProvider|Error connecting to qBittorrent', + '', + '[v4.0.10.2544] System.Net.Http.HttpRequestException: Connection refused', + ' at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync()', + '', + '2026-10-10 12:00:06.1|Info|RssSyncService|RSS Sync Completed', + ]); + + expect(entries).toHaveLength(2); + const [error, info] = entries; + expect(error?.level).toBe('error'); + expect(error?.source).toBe('DownloadClientProvider'); + expect(error?.message).toBe('Error connecting to qBittorrent - Connection refused'); + expect(error?.exception).toBe('System.Net.Http.HttpRequestException'); + expect(error?.stackTrace).toContain('HttpRequestException: Connection refused'); + expect(error?.stackTrace).toContain('ConnectToTcpHostAsync'); + expect(error?.stackTrace).not.toContain('RSS Sync'); + expect(info?.message).toBe('RSS Sync Completed'); + expect(info?.stackTrace).toBeUndefined(); + }); + + it('emits plain single-line entries', () => { + const entries = feed([ + '2026-10-10 12:00:05.4|Info|A|one', + '', + '2026-10-10 12:00:06.1|Info|B|two', + ]); + + expect(entries.map((e) => e.message)).toEqual(['one', 'two']); + }); + + it('flushes a trailing error entry at EOF with its exception', () => { + const processor = new LogFileProcessor(new SonarrProvider()); + const lines = [ + '2026-10-10 12:00:05.4|Error|Src|Boom', + '', + '[v4.0.10.2544] System.IO.IOException: disk full', + ' at Foo.Bar()', + '', + ]; + lines.forEach((l, i) => expect(processor.processLine(l, i + 1)).toBeNull()); + + const entry = processor.flush(); + expect(entry?.message).toBe('Boom - disk full'); + expect(entry?.exception).toBe('System.IO.IOException'); + expect(entry?.stackTrace).toContain('at Foo.Bar()'); + expect(processor.flush()).toBeNull(); + }); + + it('drops a stray continuation line at the start of the file', () => { + const entries = feed([' at Orphan.Frame()', '2026-10-10 12:00:05.4|Info|A|one']); + + expect(entries).toHaveLength(1); + expect(entries[0]?.message).toBe('one'); + expect(entries[0]?.stackTrace).toBeUndefined(); + }); + + it('does not attach or emit an unparseable non-continuation line between entries', () => { + // 8-digit fraction does not match the NLog format (known limitation) so it is a continuation + // of the pending entry; the surrounding entries must still be emitted intact. + const entries = feed([ + '2026-10-10 12:00:05.4|Info|A|one', + '2026-10-10 12:00:05.12345678|Info|B|weird', + '2026-10-10 12:00:06.1|Info|C|three', + ]); + + expect(entries.map((e) => e.message)).toEqual(['one', 'three']); + expect(entries[0]?.stackTrace).toContain('weird'); + }); + + it('still works for a provider without finalizeLogEntry (Emby)', () => { + const emby = new EmbyProvider(); + expect((emby as unknown as { finalizeLogEntry?: unknown }).finalizeLogEntry).toBeUndefined(); + + const entries = feed( + [ + '2024-01-15 10:30:45.123 Error App: Something failed', + ' at Foo.Bar()', + '2024-01-15 10:30:46.123 Info App: Next', + ], + emby + ); + + expect(entries).toHaveLength(2); + expect(entries[0]?.stackTrace).toContain('at Foo.Bar()'); + expect(entries[1]?.stackTrace).toBeUndefined(); + }); +}); diff --git a/apps/backend/src/modules/file-ingestion/log-file-processor.ts b/apps/backend/src/modules/file-ingestion/log-file-processor.ts index a370bdc..d280c03 100644 --- a/apps/backend/src/modules/file-ingestion/log-file-processor.ts +++ b/apps/backend/src/modules/file-ingestion/log-file-processor.ts @@ -18,6 +18,9 @@ interface LogParseResult { interface ContextAwareProvider extends MediaServerProvider { parseLogLineWithContext(line: string, context: LogParseContext): LogParseResult; isLogContinuation?(line: string): boolean; + // Optional hook to enrich an assembled multi-line entry (e.g. extract exception details). + // Only called when the entry has at least one continuation line. + finalizeLogEntry?(entry: ParsedLogEntry, continuationLines: readonly string[]): ParsedLogEntry; } function hasContextParsing(provider: MediaServerProvider): provider is ContextAwareProvider { @@ -175,7 +178,11 @@ export class LogFileProcessor { return null; } - const entry = this.pendingEntry; + let entry = this.pendingEntry; + const finalize = (this.provider as ContextAwareProvider).finalizeLogEntry; + if (typeof finalize === 'function' && this.continuationLines.length > 0) { + entry = finalize.call(this.provider, entry, this.continuationLines); + } // Clear pending entry after finalizing this.pendingEntry = null; diff --git a/packages/provider-arr/src/arr.parser.spec.ts b/packages/provider-arr/src/arr.parser.spec.ts index 27da7f8..d96f8e3 100644 --- a/packages/provider-arr/src/arr.parser.spec.ts +++ b/packages/provider-arr/src/arr.parser.spec.ts @@ -8,6 +8,7 @@ import { parseArrLogLine, isArrLogContinuation, parseArrLogLineWithContext, + finalizeArrLogEntry, extractArrMetadata, getArrSeverityBoost, ARR_LOG_FILE_CONFIG, @@ -130,6 +131,14 @@ describe('Arr Parser', () => { new Date(2026, 9, 10, 12, 0, 1, expectedMs).getTime() ); }); + + it('documents known limitation: an 8-digit fraction is not parsed as an entry and is treated as a continuation', () => { + const line = '2026-10-10 12:00:01.12345678|Info|RssSyncService|Too many digits'; + + // Regex allows at most 7 fraction digits, so the line does not match the NLog format + expect(parseArrLogLine(line)).toBeNull(); + expect(isArrLogContinuation(line)).toBe(true); + }); }); }); @@ -196,45 +205,84 @@ describe('Arr Parser', () => { }); describe('parseArrLogLineWithContext', () => { - const baseContext = { - continuationLines: [] as string[], - filePath: '/test/arr.log', - lineNumber: 1, - }; - - it('should parse a standalone entry', () => { - const line = '2024-01-15 10:30:45.123|Info|Test|Message'; - const context = { ...baseContext }; - const result = parseArrLogLineWithContext(line, context); - - // First call stores in previousEntry, returns null for entry - expect(result.entry).toBeNull(); + const context = { continuationLines: [] as string[], filePath: '/test/arr.log', lineNumber: 1 }; + + it('returns the entry parsed from the current line', () => { + const result = parseArrLogLineWithContext( + '2024-01-15 10:30:45.123|Info|Test|Message', + context + ); + + expect(result.entry?.message).toBe('Message'); expect(result.isContinuation).toBe(false); + expect(result.previousComplete).toBe(false); }); - it('should identify continuation lines', () => { - const line = ' at System.Threading.Task.Run()'; - const context = { ...baseContext }; - const result = parseArrLogLineWithContext(line, context); + it('parses one-digit fraction timestamps', () => { + const result = parseArrLogLineWithContext('2026-10-10 12:00:05.4|Error|Src|Boom', context); - expect(result.entry).toBeNull(); - expect(result.isContinuation).toBe(true); + expect(result.entry?.timestamp.getTime()).toBe( + new Date(2026, 9, 10, 12, 0, 5, 400).getTime() + ); + }); + + it('flags previousComplete when a previous entry is pending', () => { + const previousEntry = parseArrLogLine('2024-01-15 10:30:45.123|Info|Test|First')!; + const result = parseArrLogLineWithContext('2024-01-15 10:30:46.123|Info|Test|Second', { + ...context, + previousEntry, + }); + + expect(result.entry?.message).toBe('Second'); + expect(result.previousComplete).toBe(true); }); - it('should complete previous entry when new entry starts', () => { - // First line - sets previousEntry - const context = { ...baseContext, continuationLines: [] as string[] }; - parseArrLogLineWithContext('2024-01-15 10:30:45.123|Info|Test|First message', context); + it('identifies continuation lines', () => { + const result = parseArrLogLineWithContext(' at System.Threading.Task.Run()', context); + + expect(result).toEqual({ entry: null, isContinuation: true, previousComplete: false }); + }); - // Second line - should complete previous + it('treats the [vX] exception line as a continuation', () => { const result = parseArrLogLineWithContext( - '2024-01-15 10:30:46.123|Info|Test|Second message', + '[v4.0.10.2544] System.InvalidOperationException: bad state', context ); - expect(result.previousComplete).toBe(true); - expect(result.entry).not.toBeNull(); - expect(result.entry?.message).toBe('First message'); + expect(result.isContinuation).toBe(true); + }); + + it('ignores blank lines without flushing the pending entry', () => { + const previousEntry = parseArrLogLine('2026-10-10 12:00:05.4|Error|Src|Boom')!; + const result = parseArrLogLineWithContext('', { ...context, previousEntry }); + + expect(result).toEqual({ entry: null, isContinuation: false, previousComplete: false }); + }); + }); + + describe('finalizeArrLogEntry', () => { + const entry = parseArrLogLine('2026-10-10 12:00:05.4|Error|Src|Boom')!; + + it('populates exception and appends its message for a [vX] prefixed line', () => { + const result = finalizeArrLogEntry(entry, [ + '[v4.0.10.2544] System.InvalidOperationException: bad state', + ' at Foo.Bar()', + ]); + + expect(result.exception).toBe('System.InvalidOperationException'); + expect(result.message).toBe('Boom - bad state'); + }); + + it('skips leading blank lines and does not duplicate a message already present', () => { + const withMsg = parseArrLogLine('2026-10-10 12:00:05.4|Error|Src|bad state happened')!; + const result = finalizeArrLogEntry(withMsg, ['', 'System.IO.IOException: bad state']); + + expect(result.exception).toBe('System.IO.IOException'); + expect(result.message).toBe('bad state happened'); + }); + + it('returns the entry unchanged when there is no exception line', () => { + expect(finalizeArrLogEntry(entry, [' at Foo.Bar()'])).toBe(entry); }); }); diff --git a/packages/provider-arr/src/arr.parser.ts b/packages/provider-arr/src/arr.parser.ts index 1001c29..24644b0 100644 --- a/packages/provider-arr/src/arr.parser.ts +++ b/packages/provider-arr/src/arr.parser.ts @@ -175,7 +175,7 @@ export const ARR_LOG_FILE_CONFIG: LogFileConfig = { * NLog timestamp + metadata regex * Matches: 2024-01-15 10:30:45.1|Level|Component|Message * The fractional second is optional and may have 1-7 digits: *arr apps log with - * the NLog layout `yyyy-MM-dd HH:mm:ss.f` (one digit), older/custom layouts use 3. + * the NLog layout `yyyy-MM-dd HH:mm:ss.f` (one digit); custom layouts may log up to 7 digits. */ const NLOG_LINE_REGEX = /^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d{1,7})?)\|(\w+)\|([^|]+)\|(.*)$/; @@ -263,93 +263,65 @@ export function isArrLogContinuation(line: string): boolean { } /** - * Parse a log line with context for multi-line entry support + * Parse a log line with context for multi-line entry support. * - * Handles: - * - Regular single-line entries - * - Multi-line entries with stack traces - * - Exception details spanning multiple lines + * Follows the `MediaServerProvider.parseLogLineWithContext` contract: `entry` is the + * entry parsed from THIS line (not the previous one), continuation lines are flagged with + * `isContinuation` so the caller can buffer them, and `previousComplete` tells the caller + * the previously pending entry is finished. Attaching continuation lines is the caller's + * job; use `finalizeArrLogEntry` to enrich the assembled entry with exception details. */ export function parseArrLogLineWithContext(line: string, context: LogParseContext): LogParseResult { - const trimmed = line.trim(); - - // Check if this is a continuation line - if (isArrLogContinuation(line)) { - // Add to continuation buffer - context.continuationLines.push(line); - - return { - entry: null, - isContinuation: true, - previousComplete: false, - }; + // Blank lines carry no information. NLog emits one between the header and the + // exception (`${newline}${exception}`), so they must neither flush the pending + // entry nor be added to its stack trace. + if (line.trim() === '') { + return { entry: null, isContinuation: false, previousComplete: false }; } - // This is a new log entry - first, check if we need to complete the previous entry - let completedEntry: ParsedLogEntry | null = null; - - if (context.previousEntry && context.continuationLines.length > 0) { - // Check if it's an exception - const exceptionMatch = context.continuationLines[0]?.match( - /^((?:System|Microsoft|NLog|NzbDrone)\.\w+(?:\.\w+)*Exception):\s*(.*)$/i - ); - - let message = context.previousEntry.message; - let exception: string | undefined; - - if (exceptionMatch) { - exception = exceptionMatch[1]; - // Append exception message to the main message if it's different - const exceptionMessage = exceptionMatch[2]; - if ( - exceptionMessage !== undefined && - exceptionMessage !== '' && - !message.includes(exceptionMessage) - ) { - message = `${message} - ${exceptionMatch[2]}`; - } - } - - // Build the complete entry with continuation lines - completedEntry = { - ...context.previousEntry, - message, - exception, - stackTrace: context.continuationLines.join('\n'), - raw: context.previousEntry.raw + '\n' + context.continuationLines.join('\n'), - }; - } else if (context.previousEntry) { - // Previous entry is complete with no continuation - completedEntry = context.previousEntry; - } - - // Clear continuation buffer - context.continuationLines = []; - - // Parse the new line - const parsed = parseArrLogLine(trimmed); - - if (parsed) { - // Store as new previous entry - context.previousEntry = parsed; - - return { - entry: completedEntry, - isContinuation: false, - previousComplete: completedEntry !== null, - }; + if (isArrLogContinuation(line)) { + return { entry: null, isContinuation: true, previousComplete: false }; } - // Line didn't match expected format - treat as standalone - delete context.previousEntry; + const entry = parseArrLogLine(line); return { - entry: completedEntry, + entry, isContinuation: false, - previousComplete: completedEntry !== null, + previousComplete: entry !== null && context.previousEntry !== undefined, }; } +/** + * Enrich an assembled entry with exception details taken from its continuation lines. + * + * *arr prefixes the exception with its version, e.g. "[v4.0.10.2544] System.Foo: msg". + * Sets `exception` and appends the exception message to `message` when not already present. + */ +export function finalizeArrLogEntry( + entry: ParsedLogEntry, + continuationLines: readonly string[] +): ParsedLogEntry { + const first = continuationLines.find((l) => l.trim() !== ''); + const match = first + ?.trim() + .replace(/^\[v[\d.]+\]\s*/, '') + .match(/^((?:System|Microsoft|NLog|NzbDrone)\.\w+(?:\.\w+)*Exception):\s*(.*)$/i); + if (!match) return entry; + + const [, exception, exceptionMessage] = match; + if (exception === undefined || exception === '') return entry; + + const message = + exceptionMessage !== undefined && + exceptionMessage !== '' && + !entry.message.includes(exceptionMessage) + ? `${entry.message} - ${exceptionMessage}` + : entry.message; + + return { ...entry, message, exception }; +} + /** * Extract metadata from arr log messages * diff --git a/packages/provider-arr/src/arr.provider.ts b/packages/provider-arr/src/arr.provider.ts index ba476cb..9cfc771 100644 --- a/packages/provider-arr/src/arr.provider.ts +++ b/packages/provider-arr/src/arr.provider.ts @@ -7,6 +7,7 @@ import { ArrClient } from './arr.client.js'; import { parseArrLogLine, parseArrLogLineWithContext, + finalizeArrLogEntry, isArrLogContinuation, ARR_LOG_FILE_CONFIG, } from './arr.parser.js'; @@ -147,6 +148,14 @@ export abstract class ArrBaseProvider implements MediaServerProvider { return parseArrLogLineWithContext(line, context); } + /** + * Enrich an assembled multi-line entry with exception details. + * Optional hook: only called by the file processor when the entry has continuation lines. + */ + finalizeLogEntry(entry: ParsedLogEntry, continuationLines: readonly string[]): ParsedLogEntry { + return finalizeArrLogEntry(entry, continuationLines); + } + /** * Check if a line is a continuation of the previous entry */ diff --git a/packages/provider-arr/src/index.ts b/packages/provider-arr/src/index.ts index 5fe9a2f..faae45a 100644 --- a/packages/provider-arr/src/index.ts +++ b/packages/provider-arr/src/index.ts @@ -6,6 +6,7 @@ export { ArrClient } from './arr.client.js'; export { parseArrLogLine, parseArrLogLineWithContext, + finalizeArrLogEntry, isArrLogContinuation, extractArrMetadata, getArrSeverityBoost, From 39e8edcce3e79a7b8d07487b45b77103ab0bbabc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Brzezina?= Date: Sat, 10 Oct 2026 20:32:56 +0200 Subject: [PATCH 3/5] refactor(provider-arr): add shared event resolution helpers and declare finalizeLogEntry on core provider interface --- .../file-ingestion/log-file-processor.ts | 15 ++-- packages/core/src/types.ts | 7 ++ packages/provider-arr/src/arr.events.spec.ts | 81 +++++++++++++++++++ packages/provider-arr/src/arr.events.ts | 66 +++++++++++++++ packages/provider-arr/src/index.ts | 4 + 5 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 packages/provider-arr/src/arr.events.spec.ts create mode 100644 packages/provider-arr/src/arr.events.ts diff --git a/apps/backend/src/modules/file-ingestion/log-file-processor.ts b/apps/backend/src/modules/file-ingestion/log-file-processor.ts index d280c03..5ee7a09 100644 --- a/apps/backend/src/modules/file-ingestion/log-file-processor.ts +++ b/apps/backend/src/modules/file-ingestion/log-file-processor.ts @@ -18,15 +18,19 @@ interface LogParseResult { interface ContextAwareProvider extends MediaServerProvider { parseLogLineWithContext(line: string, context: LogParseContext): LogParseResult; isLogContinuation?(line: string): boolean; - // Optional hook to enrich an assembled multi-line entry (e.g. extract exception details). - // Only called when the entry has at least one continuation line. - finalizeLogEntry?(entry: ParsedLogEntry, continuationLines: readonly string[]): ParsedLogEntry; } function hasContextParsing(provider: MediaServerProvider): provider is ContextAwareProvider { return typeof (provider as ContextAwareProvider).parseLogLineWithContext === 'function'; } +type FinalizingProvider = MediaServerProvider & + Required>; + +function hasFinalizeLogEntry(provider: MediaServerProvider): provider is FinalizingProvider { + return typeof provider.finalizeLogEntry === 'function'; +} + function hasLogContinuation(provider: MediaServerProvider): provider is ContextAwareProvider { return typeof (provider as ContextAwareProvider).isLogContinuation === 'function'; } @@ -179,9 +183,8 @@ export class LogFileProcessor { } let entry = this.pendingEntry; - const finalize = (this.provider as ContextAwareProvider).finalizeLogEntry; - if (typeof finalize === 'function' && this.continuationLines.length > 0) { - entry = finalize.call(this.provider, entry, this.continuationLines); + if (hasFinalizeLogEntry(this.provider) && this.continuationLines.length > 0) { + entry = this.provider.finalizeLogEntry(entry, this.continuationLines); } // Clear pending entry after finalizing this.pendingEntry = null; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index fbc0555..06c3344 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -337,6 +337,13 @@ export interface MediaServerProvider { * (e.g., stack trace lines, wrapped messages) */ isLogContinuation?(line: string): boolean; + + /** + * Enrich an assembled multi-line entry (e.g. extract exception details from + * its continuation lines). The caller attaches the stack trace/raw text itself. + * Only called when the entry has at least one continuation line. + */ + finalizeLogEntry?(entry: ParsedLogEntry, continuationLines: readonly string[]): ParsedLogEntry; } /** diff --git a/packages/provider-arr/src/arr.events.spec.ts b/packages/provider-arr/src/arr.events.spec.ts new file mode 100644 index 0000000..adcfdcb --- /dev/null +++ b/packages/provider-arr/src/arr.events.spec.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest'; + +import { describeArrEvent, resolveArrEventName } from './arr.events.js'; + +import type { ArrEventTable } from './arr.events.js'; + +const NAMES = { 1: 'grabbed', 4: 'downloadFolderImported', 7: 'downloadFailed' } as const; + +describe('resolveArrEventName', () => { + it.each([ + [1, 'grabbed'], + [7, 'downloadFailed'], + ['4', 'downloadFolderImported'], + [' 7 ', 'downloadFailed'], + ])('maps %j to %s', (input, expected) => { + expect(resolveArrEventName(input, NAMES)).toBe(expected); + }); + + it.each([[99], ['99'], [-1], [1.5], [NaN], [Infinity]])('returns unknown for %j', (input) => { + expect(resolveArrEventName(input, NAMES)).toBe('unknown'); + }); + + it.each([[''], [' '], [null], [undefined], [{}], [[]], [true]])( + 'returns unknown for %j', + (input) => { + expect(resolveArrEventName(input, NAMES)).toBe('unknown'); + } + ); + + it('returns other strings unchanged', () => { + expect(resolveArrEventName('trackFileImported', NAMES)).toBe('trackFileImported'); + expect(resolveArrEventName('Grabbed', NAMES)).toBe('Grabbed'); + }); + + it('does not resolve inherited object keys', () => { + expect(resolveArrEventName('constructor', {})).toBe('constructor'); + expect(resolveArrEventName(0, {})).toBe('unknown'); + }); +}); + +describe('describeArrEvent', () => { + const table: ArrEventTable = { + downloadFailed: { label: 'Download Failed', severity: 'error' }, + albumImportIncomplete: { label: 'Import Incomplete', severity: 'warn' }, + grabbed: { label: 'Grabbed', severity: 'info' }, + }; + + it('returns the table entry for a known name', () => { + expect(describeArrEvent('albumImportIncomplete', table)).toEqual({ + label: 'Import Incomplete', + severity: 'warn', + }); + }); + + it('matches names case-insensitively', () => { + expect(describeArrEvent('DOWNLOADFAILED', table)).toBe(table['downloadFailed']); + }); + + it('defaults unknown names containing "fail" to error', () => { + expect(describeArrEvent('bookImportFailed', table)).toEqual({ + label: 'Book Import Failed', + severity: 'error', + }); + }); + + it('defaults other unknown names to info with a humanized label', () => { + expect(describeArrEvent('bookFileRetagged', table)).toEqual({ + label: 'Book File Retagged', + severity: 'info', + }); + }); + + it('handles the unknown sentinel and empty table', () => { + expect(describeArrEvent('unknown', {})).toEqual({ label: 'Unknown', severity: 'info' }); + expect(describeArrEvent('', {})).toEqual({ label: 'Unknown', severity: 'info' }); + }); + + it('does not resolve inherited object keys', () => { + expect(describeArrEvent('constructor', {}).severity).toBe('info'); + }); +}); diff --git a/packages/provider-arr/src/arr.events.ts b/packages/provider-arr/src/arr.events.ts new file mode 100644 index 0000000..d73cfe2 --- /dev/null +++ b/packages/provider-arr/src/arr.events.ts @@ -0,0 +1,66 @@ +/** + * Shared helpers for resolving and describing *arr history event types. + * + * *arr APIs report `eventType` either as an enum name ("downloadFailed"), as a + * numeric enum value (4) or as a numeric string ("4"), depending on the app/version. + * Per-app providers supply their own enum table and display table. + */ + +export type ArrEventSeverity = 'info' | 'warn' | 'error'; + +/** Display label and severity for a single event */ +export interface ArrEventDescriptor { + readonly label: string; + readonly severity: ArrEventSeverity; +} + +/** Event name (camelCase, as reported by the API) to descriptor */ +export type ArrEventTable = Readonly>; + +/** + * Resolve a raw `eventType` to an event name. + * + * - integer number or numeric string: looked up in `names`; absent -> 'unknown' + * - any other non-blank string: returned unchanged + * - blank string, non-string/non-number, NaN/Infinity, fractional: 'unknown' + */ +export function resolveArrEventName( + eventType: unknown, + names: Readonly> +): string { + if (typeof eventType === 'number') { + return Number.isInteger(eventType) ? (names[eventType] ?? 'unknown') : 'unknown'; + } + + if (typeof eventType === 'string') { + const trimmed = eventType.trim(); + if (trimmed === '') return 'unknown'; + if (/^\d+$/.test(trimmed)) return names[Number(trimmed)] ?? 'unknown'; + return eventType; + } + + return 'unknown'; +} + +function humanize(name: string): string { + const spaced = name.replace(/([a-z0-9])([A-Z])/g, '$1 $2').trim(); + return spaced === '' ? 'Unknown' : spaced.charAt(0).toUpperCase() + spaced.slice(1); +} + +/** + * Describe an event by name using a per-app table. + * + * Lookup is exact first, then case-insensitive. Names missing from the table get + * a humanized label and a default severity: 'error' when the name contains "fail", + * otherwise 'info'. + */ +export function describeArrEvent(name: string, table: ArrEventTable): ArrEventDescriptor { + if (Object.hasOwn(table, name)) return table[name] as ArrEventDescriptor; + + const lower = name.toLowerCase(); + for (const [key, descriptor] of Object.entries(table)) { + if (key.toLowerCase() === lower) return descriptor; + } + + return { label: humanize(name), severity: /fail/i.test(name) ? 'error' : 'info' }; +} diff --git a/packages/provider-arr/src/index.ts b/packages/provider-arr/src/index.ts index faae45a..0b90853 100644 --- a/packages/provider-arr/src/index.ts +++ b/packages/provider-arr/src/index.ts @@ -2,6 +2,10 @@ export { ArrBaseProvider } from './arr.provider.js'; export { ArrClient } from './arr.client.js'; +// Shared event name resolution and display helpers +export { resolveArrEventName, describeArrEvent } from './arr.events.js'; +export type { ArrEventSeverity, ArrEventDescriptor, ArrEventTable } from './arr.events.js'; + // Parser utilities for file-based log ingestion export { parseArrLogLine, From 5fcfd5ffa79e98489d5f9b6d3caafaacaa44bbdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Brzezina?= Date: Sat, 10 Oct 2026 20:42:22 +0200 Subject: [PATCH 4/5] refactor(provider-arr): canonicalise event name casing and add toArrItemId helper --- packages/provider-arr/src/arr.events.spec.ts | 27 +++++++++++++++++--- packages/provider-arr/src/arr.events.ts | 16 ++++++++++-- packages/provider-arr/src/index.ts | 2 +- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/provider-arr/src/arr.events.spec.ts b/packages/provider-arr/src/arr.events.spec.ts index adcfdcb..83aae99 100644 --- a/packages/provider-arr/src/arr.events.spec.ts +++ b/packages/provider-arr/src/arr.events.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { describeArrEvent, resolveArrEventName } from './arr.events.js'; +import { describeArrEvent, resolveArrEventName, toArrItemId } from './arr.events.js'; import type { ArrEventTable } from './arr.events.js'; @@ -27,9 +27,14 @@ describe('resolveArrEventName', () => { } ); - it('returns other strings unchanged', () => { + it('returns unmatched strings unchanged', () => { expect(resolveArrEventName('trackFileImported', NAMES)).toBe('trackFileImported'); - expect(resolveArrEventName('Grabbed', NAMES)).toBe('Grabbed'); + }); + + it('matches known names case-insensitively', () => { + expect(resolveArrEventName('Grabbed', NAMES)).toBe('grabbed'); + expect(resolveArrEventName('DownloadFailed', NAMES)).toBe('downloadFailed'); + expect(resolveArrEventName(' DOWNLOADFAILED ', NAMES)).toBe('downloadFailed'); }); it('does not resolve inherited object keys', () => { @@ -79,3 +84,19 @@ describe('describeArrEvent', () => { expect(describeArrEvent('constructor', {}).severity).toBe('info'); }); }); + +describe('toArrItemId', () => { + it.each([ + [1, '1'], + [42, '42'], + ])('stringifies %j', (input, expected) => { + expect(toArrItemId(input)).toBe(expected); + }); + + it.each([[0], [-1], [1.5], [NaN], [Infinity], ['5'], [null], [undefined], [{}]])( + 'returns undefined for %j', + (input) => { + expect(toArrItemId(input)).toBeUndefined(); + } + ); +}); diff --git a/packages/provider-arr/src/arr.events.ts b/packages/provider-arr/src/arr.events.ts index d73cfe2..6f65bdc 100644 --- a/packages/provider-arr/src/arr.events.ts +++ b/packages/provider-arr/src/arr.events.ts @@ -21,7 +21,8 @@ export type ArrEventTable = Readonly>; * Resolve a raw `eventType` to an event name. * * - integer number or numeric string: looked up in `names`; absent -> 'unknown' - * - any other non-blank string: returned unchanged + * - any other non-blank string: matched case-insensitively against the known names in + * `names` (so 'DownloadFailed' -> 'downloadFailed'); if none match, returned unchanged * - blank string, non-string/non-number, NaN/Infinity, fractional: 'unknown' */ export function resolveArrEventName( @@ -36,12 +37,23 @@ export function resolveArrEventName( const trimmed = eventType.trim(); if (trimmed === '') return 'unknown'; if (/^\d+$/.test(trimmed)) return names[Number(trimmed)] ?? 'unknown'; - return eventType; + const lower = trimmed.toLowerCase(); + const known = Object.values(names).find((name) => name.toLowerCase() === lower); + return known ?? eventType; } return 'unknown'; } +/** + * Convert a raw record id to an item id string. + * Returns the stringified id only for positive finite integers (numbers only; strings + * are not accepted), otherwise undefined (e.g. 0, -1, NaN, null). + */ +export function toArrItemId(id: unknown): string | undefined { + return typeof id === 'number' && Number.isInteger(id) && id > 0 ? String(id) : undefined; +} + function humanize(name: string): string { const spaced = name.replace(/([a-z0-9])([A-Z])/g, '$1 $2').trim(); return spaced === '' ? 'Unknown' : spaced.charAt(0).toUpperCase() + spaced.slice(1); diff --git a/packages/provider-arr/src/index.ts b/packages/provider-arr/src/index.ts index 0b90853..0fcde3c 100644 --- a/packages/provider-arr/src/index.ts +++ b/packages/provider-arr/src/index.ts @@ -3,7 +3,7 @@ export { ArrBaseProvider } from './arr.provider.js'; export { ArrClient } from './arr.client.js'; // Shared event name resolution and display helpers -export { resolveArrEventName, describeArrEvent } from './arr.events.js'; +export { resolveArrEventName, describeArrEvent, toArrItemId } from './arr.events.js'; export type { ArrEventSeverity, ArrEventDescriptor, ArrEventTable } from './arr.events.js'; // Parser utilities for file-based log ingestion From 08082073c2f4fa34163a069b3be1e280d6ae37b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Brzezina?= Date: Mon, 12 Oct 2026 18:18:24 +0200 Subject: [PATCH 5/5] fix(file-ingestion): skip *arr debug/trace rolling logs Now that *arr file ingestion works, the shared *.txt pattern also picked up the apps' rolling debug and trace logs (Radarr keeps 50+ radarr.debug.N.txt files), duplicating the main log at higher verbosity. Backfilling them exhausted a 384 MB backend's Node heap and caused Postgres deadlocks from dozens of parallel tailers. - LogFileConfig gains optional excludePatterns (core) - every *arr log config excludes *.debug.* and *.trace.* - FileDiscoveryService skips excluded basenames; servers with custom logFilePatterns get no excludes, so debug logs can still be opted into --- .../file-discovery.service.spec.ts | 82 +++++++++++++++++ .../file-ingestion/file-discovery.service.ts | 36 ++++++-- .../file-ingestion.service.spec.ts | 88 +++++++++++++++++++ .../file-ingestion/file-ingestion.service.ts | 19 +++- packages/core/src/types.ts | 2 + packages/provider-arr/src/arr.parser.spec.ts | 21 +++++ packages/provider-arr/src/arr.parser.ts | 16 ++++ packages/provider-arr/src/arr.provider.ts | 1 + packages/provider-arr/src/index.ts | 1 + 9 files changed, 259 insertions(+), 7 deletions(-) diff --git a/apps/backend/src/modules/file-ingestion/file-discovery.service.spec.ts b/apps/backend/src/modules/file-ingestion/file-discovery.service.spec.ts index ea1e1cd..35ed359 100644 --- a/apps/backend/src/modules/file-ingestion/file-discovery.service.spec.ts +++ b/apps/backend/src/modules/file-ingestion/file-discovery.service.spec.ts @@ -1,6 +1,15 @@ import { existsSync, readdirSync, statSync } from 'fs'; import { platform, homedir } from 'os'; +import { + ARR_LOG_FILE_CONFIG, + LIDARR_LOG_FILE_CONFIG, + PROWLARR_LOG_FILE_CONFIG, + RADARR_LOG_FILE_CONFIG, + READARR_LOG_FILE_CONFIG, + SONARR_LOG_FILE_CONFIG, + WHISPARR_LOG_FILE_CONFIG, +} from '@logarr/provider-arr'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FileDiscoveryService } from './file-discovery.service'; @@ -288,6 +297,79 @@ describe('FileDiscoveryService', () => { }); describe('discoverLogFiles', () => { + describe('exclude patterns', () => { + const mockDir = (names: string[]): void => { + vi.mocked(existsSync).mockReturnValue(true); + vi.mocked(statSync).mockReturnValue({ + isDirectory: () => true, + isFile: () => false, + mtimeMs: Date.now(), + } as ReturnType); + vi.mocked(readdirSync).mockReturnValue( + names.map((name) => ({ + name, + isFile: () => true, + isDirectory: () => false, + })) as unknown as ReturnType + ); + }; + + it('skips files matching an exclude pattern', async () => { + mockDir(['app.txt', 'app.debug.txt']); + + const files = await service.discoverLogFiles(['/logs'], ['*.txt'], ['*.debug.*']); + + expect(files).toEqual(['/logs/app.txt']); + }); + + it('does not exclude names that merely start with debug/trace', async () => { + mockDir(['foo.debugger.txt', 'foo.tracer.txt', 'foo.debug.txt']); + + const files = await service.discoverLogFiles( + ['/logs'], + ['*.txt'], + ['*.debug.*', '*.trace.*'] + ); + + expect(files.map((f) => f.replace('/logs/', '')).sort()).toEqual([ + 'foo.debugger.txt', + 'foo.tracer.txt', + ]); + }); + + it.each([ + ['sonarr', SONARR_LOG_FILE_CONFIG], + ['radarr', RADARR_LOG_FILE_CONFIG], + ['prowlarr', PROWLARR_LOG_FILE_CONFIG], + ['whisparr', WHISPARR_LOG_FILE_CONFIG], + ['lidarr', LIDARR_LOG_FILE_CONFIG], + ['readarr', READARR_LOG_FILE_CONFIG], + ['arr', ARR_LOG_FILE_CONFIG], + ])( + '%s config ingests main and rotated logs but not debug/trace logs', + async (app, config) => { + const name = app === 'arr' ? 'radarr' : app; + const wanted = [`${name}.txt`, `${name}.0.txt`, `${name}.1.txt`, `${name}.9.txt`]; + const unwanted = [ + `${name}.debug.txt`, + `${name}.debug.0.txt`, + `${name}.debug.49.txt`, + `${name}.trace.txt`, + `${name}.trace.3.txt`, + ]; + mockDir([...wanted, ...unwanted]); + + const files = await service.discoverLogFiles( + ['/logs'], + [...config.filePatterns], + [...(config.excludePatterns ?? [])] + ); + + expect(files.map((f) => f.replace('/logs/', '')).sort()).toEqual([...wanted].sort()); + } + ); + }); + it('should skip non-existent paths', async () => { vi.mocked(existsSync).mockReturnValue(false); diff --git a/apps/backend/src/modules/file-ingestion/file-discovery.service.ts b/apps/backend/src/modules/file-ingestion/file-discovery.service.ts index 3b11e39..5669ec7 100644 --- a/apps/backend/src/modules/file-ingestion/file-discovery.service.ts +++ b/apps/backend/src/modules/file-ingestion/file-discovery.service.ts @@ -13,6 +13,7 @@ interface LogFileConfig { macos: readonly string[]; }; filePatterns: readonly string[]; + excludePatterns?: readonly string[]; encoding?: string; rotatesDaily?: boolean; datePattern?: RegExp; @@ -58,8 +59,13 @@ export class FileDiscoveryService { /** * Discover log files in the given paths matching the patterns + * (files matching any exclude pattern are skipped) */ - async discoverLogFiles(paths: string[], patterns: string[]): Promise { + async discoverLogFiles( + paths: string[], + patterns: string[], + excludePatterns: string[] = [] + ): Promise { const discoveredFiles: string[] = []; for (const basePath of paths) { @@ -75,11 +81,15 @@ export class FileDiscoveryService { if (stats.isDirectory()) { // Discover files in directory - const files = await this.discoverFilesInDirectory(expandedPath, patterns); + const files = await this.discoverFilesInDirectory( + expandedPath, + patterns, + excludePatterns + ); discoveredFiles.push(...files); } else if (stats.isFile()) { // Direct file path - if (this.matchesPatterns(expandedPath, patterns)) { + if (this.isIncluded(expandedPath, patterns, excludePatterns)) { discoveredFiles.push(expandedPath); } } @@ -96,7 +106,11 @@ export class FileDiscoveryService { /** * Discover files in a directory matching patterns */ - private async discoverFilesInDirectory(dirPath: string, patterns: string[]): Promise { + private async discoverFilesInDirectory( + dirPath: string, + patterns: string[], + excludePatterns: string[] + ): Promise { const files: string[] = []; try { @@ -106,7 +120,7 @@ export class FileDiscoveryService { const fullPath = join(dirPath, entry.name); if (entry.isFile()) { - if (this.matchesPatterns(entry.name, patterns)) { + if (this.isIncluded(entry.name, patterns, excludePatterns)) { files.push(fullPath); } } else if (entry.isDirectory()) { @@ -121,6 +135,18 @@ export class FileDiscoveryService { return files; } + /** + * A file is included when it matches an include pattern and no exclude pattern. + * Patterns are matched against the base name (a direct file path is reduced to it). + */ + private isIncluded(filename: string, patterns: string[], excludePatterns: string[]): boolean { + const baseName = filename.split(/[/\\]/).pop() ?? filename; + if (excludePatterns.length > 0 && this.matchesPatterns(baseName, excludePatterns)) { + return false; + } + return this.matchesPatterns(filename, patterns); + } + /** * Check if a filename matches any of the patterns */ diff --git a/apps/backend/src/modules/file-ingestion/file-ingestion.service.spec.ts b/apps/backend/src/modules/file-ingestion/file-ingestion.service.spec.ts index afbdc11..ed7eac8 100644 --- a/apps/backend/src/modules/file-ingestion/file-ingestion.service.spec.ts +++ b/apps/backend/src/modules/file-ingestion/file-ingestion.service.spec.ts @@ -571,6 +571,94 @@ describe('FileIngestionService', () => { }); }); + describe('exclude pattern wiring', () => { + const logFileConfig = { + filePatterns: ['radarr.txt', 'radarr.*.txt'], + excludePatterns: ['*.debug.*', '*.trace.*'], + }; + const providerFor = (): Map => + new Map([['arr', { getLogFileConfig: vi.fn().mockReturnValue(logFileConfig) }]]); + + const baseServer = { + id: 'server-1', + name: 'Radarr', + providerId: 'arr', + logPaths: ['/logs'], + logFilePatterns: null as string[] | null, + fileIngestionEnabled: true, + initialSyncCompleted: false, + }; + + beforeEach(() => { + mockFileDiscoveryService.discoverLogFiles.mockResolvedValue([]); + mockFileDiscoveryService.validatePath.mockResolvedValue({ accessible: true }); + mockSettingsService.getFileIngestionSettings.mockRejectedValue(new Error('use defaults')); + mockDb.update.mockReturnValue({ + set: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }), + }); + }); + + const mockSelectServer = (server: typeof baseServer): void => { + mockDb.select.mockReturnValue({ + from: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue([server]) }), + }); + }; + + describe('startServerFileIngestion', () => { + beforeEach(() => { + service.validateLogPaths = vi.fn().mockResolvedValue({ valid: true, results: [] }); + }); + + it('passes the provider excludePatterns to discovery', async () => { + await service.startServerFileIngestion(baseServer as any, providerFor() as any); + + expect(mockFileDiscoveryService.discoverLogFiles).toHaveBeenCalledWith( + ['/logs'], + ['radarr.txt', 'radarr.*.txt'], + ['*.debug.*', '*.trace.*'] + ); + }); + + it('passes no excludes when the server has custom logFilePatterns', async () => { + const server = { ...baseServer, logFilePatterns: ['*.txt'] }; + + await service.startServerFileIngestion(server as any, providerFor() as any); + + expect(mockFileDiscoveryService.discoverLogFiles).toHaveBeenCalledWith( + ['/logs'], + ['*.txt'], + [] + ); + }); + }); + + describe('backfillFromFiles', () => { + it('passes the provider excludePatterns to discovery', async () => { + mockSelectServer(baseServer); + + await service.backfillFromFiles('server-1', providerFor() as any); + + expect(mockFileDiscoveryService.discoverLogFiles).toHaveBeenCalledWith( + ['/logs'], + ['radarr.txt', 'radarr.*.txt'], + ['*.debug.*', '*.trace.*'] + ); + }); + + it('passes no excludes when the server has custom logFilePatterns', async () => { + mockSelectServer({ ...baseServer, logFilePatterns: ['*.txt'] }); + + await service.backfillFromFiles('server-1', providerFor() as any); + + expect(mockFileDiscoveryService.discoverLogFiles).toHaveBeenCalledWith( + ['/logs'], + ['*.txt'], + [] + ); + }); + }); + }); + describe('error handling and edge cases', () => { it('should handle settings loading failure', async () => { mockSettingsService.getFileIngestionSettings.mockRejectedValue(new Error('Settings error')); diff --git a/apps/backend/src/modules/file-ingestion/file-ingestion.service.ts b/apps/backend/src/modules/file-ingestion/file-ingestion.service.ts index 52ea3a7..b3eea85 100644 --- a/apps/backend/src/modules/file-ingestion/file-ingestion.service.ts +++ b/apps/backend/src/modules/file-ingestion/file-ingestion.service.ts @@ -73,6 +73,7 @@ interface LogFileConfig { macos: readonly string[]; }; filePatterns: readonly string[]; + excludePatterns?: readonly string[]; encoding?: string; rotatesDaily?: boolean; datePattern?: RegExp; @@ -404,6 +405,11 @@ export class FileIngestionService implements OnModuleInit, OnModuleDestroy { server.logFilePatterns && server.logFilePatterns.length > 0 ? [...server.logFilePatterns] : [...logFileConfig.filePatterns]; + // Exclusions come from the provider config; explicit user patterns are taken as-is + const excludePatterns = + server.logFilePatterns && server.logFilePatterns.length > 0 + ? [] + : [...(logFileConfig.excludePatterns ?? [])]; this.logger.log(`Starting file ingestion for ${server.name}`); this.logger.log(` Paths: ${paths.join(', ')}`); @@ -448,7 +454,11 @@ export class FileIngestionService implements OnModuleInit, OnModuleDestroy { } // Discover log files - const allLogFiles = await this.fileDiscoveryService.discoverLogFiles(paths, patterns); + const allLogFiles = await this.fileDiscoveryService.discoverLogFiles( + paths, + patterns, + excludePatterns + ); if (allLogFiles.length === 0) { this.logger.warn( @@ -1258,11 +1268,16 @@ export class FileIngestionService implements OnModuleInit, OnModuleDestroy { server.logFilePatterns !== null && server.logFilePatterns.length > 0 ? [...server.logFilePatterns] : [...logFileConfig.filePatterns]; + const excludePatterns = + server.logFilePatterns !== null && server.logFilePatterns.length > 0 + ? [] + : [...(logFileConfig.excludePatterns ?? [])]; // Discover all log files const logFiles = await this.fileDiscoveryService.discoverLogFiles( [...server.logPaths], - patterns + patterns, + excludePatterns ); if (logFiles.length === 0) { diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 06c3344..a55b7e3 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -101,6 +101,8 @@ export interface LogFileConfig { }; /** File name patterns to match (glob patterns) */ readonly filePatterns: readonly string[]; + /** File name patterns (glob) to skip even if they match filePatterns */ + readonly excludePatterns?: readonly string[] | undefined; /** File encoding (default: utf-8) */ readonly encoding?: string | undefined; /** Whether log files rotate daily */ diff --git a/packages/provider-arr/src/arr.parser.spec.ts b/packages/provider-arr/src/arr.parser.spec.ts index d96f8e3..203a313 100644 --- a/packages/provider-arr/src/arr.parser.spec.ts +++ b/packages/provider-arr/src/arr.parser.spec.ts @@ -12,6 +12,7 @@ import { extractArrMetadata, getArrSeverityBoost, ARR_LOG_FILE_CONFIG, + ARR_LOG_EXCLUDE_PATTERNS, SONARR_LOG_FILE_CONFIG, RADARR_LOG_FILE_CONFIG, LIDARR_LOG_FILE_CONFIG, @@ -366,6 +367,26 @@ describe('Arr Parser', () => { }); }); +describe('ARR_LOG_EXCLUDE_PATTERNS', () => { + const configs = { + ARR_LOG_FILE_CONFIG, + SONARR_LOG_FILE_CONFIG, + RADARR_LOG_FILE_CONFIG, + LIDARR_LOG_FILE_CONFIG, + READARR_LOG_FILE_CONFIG, + PROWLARR_LOG_FILE_CONFIG, + WHISPARR_LOG_FILE_CONFIG, + }; + + it.each(Object.entries(configs))('%s excludes debug and trace logs', (_name, config) => { + expect(config.excludePatterns).toEqual(ARR_LOG_EXCLUDE_PATTERNS); + }); + + it('lists debug and trace patterns', () => { + expect(ARR_LOG_EXCLUDE_PATTERNS).toEqual(['*.debug.*', '*.trace.*']); + }); +}); + describe('Log File Configurations', () => { describe('ARR_LOG_FILE_CONFIG (generic)', () => { it('should have valid default paths', () => { diff --git a/packages/provider-arr/src/arr.parser.ts b/packages/provider-arr/src/arr.parser.ts index 24644b0..b744a51 100644 --- a/packages/provider-arr/src/arr.parser.ts +++ b/packages/provider-arr/src/arr.parser.ts @@ -18,6 +18,7 @@ interface LogFileConfig { macos: readonly string[]; }; filePatterns: readonly string[]; + excludePatterns?: readonly string[]; encoding?: string; rotatesDaily?: boolean; datePattern?: RegExp; @@ -36,6 +37,14 @@ interface LogParseResult { previousComplete: boolean; } +/** + * *arr apps also write rolling debug/trace logs next to the main log + * (e.g. radarr.debug.txt, radarr.debug.0.txt ... radarr.trace.3.txt). These duplicate the + * main log at higher verbosity, so file ingestion must skip them. Only the main log + * (.txt) and its numbered rotations (.N.txt) are ingested. + */ +export const ARR_LOG_EXCLUDE_PATTERNS: readonly string[] = ['*.debug.*', '*.trace.*']; + /** * Log file configuration for Sonarr */ @@ -51,6 +60,7 @@ export const SONARR_LOG_FILE_CONFIG: LogFileConfig = { macos: ['~/.config/Sonarr/logs', '~/Library/Application Support/Sonarr/logs'], }, filePatterns: ['sonarr.txt', 'sonarr.*.txt', '*.log'], + excludePatterns: ARR_LOG_EXCLUDE_PATTERNS, encoding: 'utf-8', rotatesDaily: true, datePattern: /sonarr\.(\d{4}-\d{2}-\d{2})\.txt$/, @@ -71,6 +81,7 @@ export const RADARR_LOG_FILE_CONFIG: LogFileConfig = { macos: ['~/.config/Radarr/logs', '~/Library/Application Support/Radarr/logs'], }, filePatterns: ['radarr.txt', 'radarr.*.txt', '*.log'], + excludePatterns: ARR_LOG_EXCLUDE_PATTERNS, encoding: 'utf-8', rotatesDaily: true, datePattern: /radarr\.(\d{4}-\d{2}-\d{2})\.txt$/, @@ -91,6 +102,7 @@ export const LIDARR_LOG_FILE_CONFIG: LogFileConfig = { macos: ['~/.config/Lidarr/logs', '~/Library/Application Support/Lidarr/logs'], }, filePatterns: ['lidarr.txt', 'lidarr.*.txt', '*.log'], + excludePatterns: ARR_LOG_EXCLUDE_PATTERNS, encoding: 'utf-8', rotatesDaily: true, datePattern: /lidarr\.(\d{4}-\d{2}-\d{2})\.txt$/, @@ -111,6 +123,7 @@ export const READARR_LOG_FILE_CONFIG: LogFileConfig = { macos: ['~/.config/Readarr/logs', '~/Library/Application Support/Readarr/logs'], }, filePatterns: ['readarr.txt', 'readarr.*.txt', '*.log'], + excludePatterns: ARR_LOG_EXCLUDE_PATTERNS, encoding: 'utf-8', rotatesDaily: true, datePattern: /readarr\.(\d{4}-\d{2}-\d{2})\.txt$/, @@ -131,6 +144,7 @@ export const PROWLARR_LOG_FILE_CONFIG: LogFileConfig = { macos: ['~/.config/Prowlarr/logs', '~/Library/Application Support/Prowlarr/logs'], }, filePatterns: ['prowlarr.txt', 'prowlarr.*.txt', '*.log'], + excludePatterns: ARR_LOG_EXCLUDE_PATTERNS, encoding: 'utf-8', rotatesDaily: true, datePattern: /prowlarr\.(\d{4}-\d{2}-\d{2})\.txt$/, @@ -151,6 +165,7 @@ export const WHISPARR_LOG_FILE_CONFIG: LogFileConfig = { macos: ['~/.config/Whisparr/logs', '~/Library/Application Support/Whisparr/logs'], }, filePatterns: ['whisparr.txt', 'whisparr.*.txt', '*.log'], + excludePatterns: ARR_LOG_EXCLUDE_PATTERNS, encoding: 'utf-8', rotatesDaily: true, datePattern: /whisparr\.(\d{4}-\d{2}-\d{2})\.txt$/, @@ -167,6 +182,7 @@ export const ARR_LOG_FILE_CONFIG: LogFileConfig = { macos: ['~/.config/*/logs'], }, filePatterns: ['*.txt', '*.log'], + excludePatterns: ARR_LOG_EXCLUDE_PATTERNS, encoding: 'utf-8', rotatesDaily: true, }; diff --git a/packages/provider-arr/src/arr.provider.ts b/packages/provider-arr/src/arr.provider.ts index 9cfc771..20b3961 100644 --- a/packages/provider-arr/src/arr.provider.ts +++ b/packages/provider-arr/src/arr.provider.ts @@ -40,6 +40,7 @@ interface LogFileConfig { macos: readonly string[]; }; filePatterns: readonly string[]; + excludePatterns?: readonly string[]; encoding?: string; rotatesDaily?: boolean; datePattern?: RegExp; diff --git a/packages/provider-arr/src/index.ts b/packages/provider-arr/src/index.ts index 0fcde3c..5a70b97 100644 --- a/packages/provider-arr/src/index.ts +++ b/packages/provider-arr/src/index.ts @@ -15,6 +15,7 @@ export { extractArrMetadata, getArrSeverityBoost, ARR_LOG_FILE_CONFIG, + ARR_LOG_EXCLUDE_PATTERNS, SONARR_LOG_FILE_CONFIG, RADARR_LOG_FILE_CONFIG, LIDARR_LOG_FILE_CONFIG,