From 8bb1dff9889fc1a6a23cf12b98eaee45f4e22106 Mon Sep 17 00:00:00 2001 From: Vitor Castro Date: Mon, 17 Aug 2026 23:19:34 +0100 Subject: [PATCH 1/2] Auto-refresh wiki glossary from digests and expand acronyms in search. Digest rebuild now harvests acronym pairs into wiki:glossary, preserves manual overrides, and resolves bidirectional alias lookup in wiki_search without embeddings. --- .../Prompts/AgenticToolDescriptionBuilder.cs | 4 +- .../GlobalWiki/GlobalWikiAliasLexicon.cs | 521 ++++++++++++++++++ .../GlobalWiki/GlobalWikiCatalog.cs | 10 + .../GlobalWiki/GlobalWikiDigestGenerator.cs | 2 +- .../GlobalWiki/GlobalWikiScoring.cs | 44 +- .../GlobalWiki/GlobalWikiService.cs | 83 ++- .../Localization/LlmPrompts.cs | 12 +- .../Models/GlobalWikiModels.cs | 6 + .../Postgres/PostgresGlobalWikiStore.cs | 72 ++- .../Wiki/FileGlobalWikiStore.cs | 9 +- .../GlobalWikiAliasLexiconTests.cs | 142 +++++ .../GlobalWikiServiceTests.cs | 156 +++++- 12 files changed, 1017 insertions(+), 44 deletions(-) create mode 100644 src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs create mode 100644 tests/ContextMemory.Api.Tests/GlobalWikiAliasLexiconTests.cs diff --git a/src/ContextMemory.Core/Agentic/Prompts/AgenticToolDescriptionBuilder.cs b/src/ContextMemory.Core/Agentic/Prompts/AgenticToolDescriptionBuilder.cs index fca50c8..0a1b694 100644 --- a/src/ContextMemory.Core/Agentic/Prompts/AgenticToolDescriptionBuilder.cs +++ b/src/ContextMemory.Core/Agentic/Prompts/AgenticToolDescriptionBuilder.cs @@ -116,8 +116,8 @@ public static string BuildMcpDescription(McpToolDefinition tool, AppRuntimeConfi public static string BuildWikiSearchDescription(AppRuntimeConfig config) => TenantLocale.Select( config.DefaultLanguage, - "Search the app's global knowledge base (Jira, Confluence, SQL exports, and other ingested documents). Use when the question needs documented facts not present in session memory. Optional asOf (ISO-8601) retrieves what was valid at that time. Do not use for greetings or pure conversational replies.", - "Pesquisa a base de conhecimento global da app (Jira, Confluence, exports SQL e outros documentos ingeridos). Usa quando a pergunta precisar de factos documentados que não estão na memória da sessão. asOf opcional (ISO-8601) devolve o que era válido nessa data. Não uses para saudações ou conversa pura."); + "Search the app's global knowledge base (Jira, Confluence, SQL exports, and other ingested documents). Use when the question needs documented facts not present in session memory. Acronyms resolve from digests and the auto-refreshed wiki glossary (e.g. VVE ↔ Virtual Vehicle Enablement). Optional asOf (ISO-8601) retrieves what was valid at that time. Do not use for greetings or pure conversational replies.", + "Pesquisa a base de conhecimento global da app (Jira, Confluence, exports SQL e outros documentos ingeridos). Usa quando a pergunta precisar de factos documentados que não estão na memória da sessão. Siglas resolvem-se a partir dos digests e do wiki:glossary auto-atualizado (ex. VVE ↔ Virtual Vehicle Enablement). asOf opcional (ISO-8601) devolve o que era válido nessa data. Não uses para saudações ou conversa pura."); private static bool IsSelfHosted(ExecutionToolConfig? execution) => execution is not null diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs new file mode 100644 index 0000000..71acc45 --- /dev/null +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs @@ -0,0 +1,521 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.RegularExpressions; +using ContextMemory.Core.Models; + +namespace ContextMemory.Core.GlobalWiki; + +/// +/// Bidirectional acronym ↔ expansion map for lexical wiki search (not embeddings). +/// Auto pairs come from digests; manual overrides live under ## Manual overrides in wiki:glossary. +/// +public sealed class GlobalWikiAliasLexicon +{ + public static GlobalWikiAliasLexicon Empty { get; } = new(); + + private static readonly ConcurrentDictionary LastByApp = + new(StringComparer.OrdinalIgnoreCase); + + private static readonly HashSet Stopwords = new(StringComparer.OrdinalIgnoreCase) + { + "a", "an", "and", "as", "at", "by", "da", "das", "de", "do", "dos", "e", "for", + "from", "in", "of", "on", "or", "o", "os", "the", "to", "um", "uma" + }; + + private static readonly HashSet BlockedAcronyms = new(StringComparer.OrdinalIgnoreCase) + { + "AND", "ARE", "BUT", "FOR", "HTTP", "HTTPS", "JSON", "NOTE", "NULL", "THE", "TODO", "YOU" + }; + + private static readonly Regex Parenthetical = new( + @"\b([A-Z][A-Z0-9]{1,7})\s*\(([^)]{3,80})\)", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex EmDashPair = new( + @"^(?:#{1,6}\s+)?(?:[-*]\s+)?([A-Z][A-Z0-9]{1,7})\s*[—–\-]\s+(.{4,80})$", + RegexOptions.CultureInvariant | RegexOptions.Compiled | RegexOptions.Multiline); + + private static readonly Regex GlossaryLine = new( + @"^(?:[-*]\s+)?(?:\|\s*)?([A-Za-z][A-Za-z0-9]{1,7})\s*(?:[:|—–]|\s-\s)\s*(.+?)(?:\s*\|)?\s*$", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex AliasesLine = new( + @"^Aliases:\s*(.+)$", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled | RegexOptions.Multiline); + + public const string AutoSectionHeader = "## Auto"; + public const string ManualSectionHeader = "## Manual overrides"; + + private readonly Dictionary _acronymToExpansion = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _expansionToAcronym = new(StringComparer.OrdinalIgnoreCase); + + public int PairCount => _acronymToExpansion.Count; + + public IReadOnlyDictionary ExportPairs() => + new Dictionary(_acronymToExpansion, StringComparer.OrdinalIgnoreCase); + + /// Harvest acronym pairs from document titles and digests (not full bodies). + public static IReadOnlyDictionary HarvestPairsFromDocuments( + IEnumerable documents) + { + var lexicon = new GlobalWikiAliasLexicon(); + foreach (var doc in documents) + { + if (GlobalWikiCatalog.IsReservedDocument(doc.DocumentId)) + continue; + lexicon.Harvest(doc.Title, doc.Summary); + } + + return lexicon.ExportPairs(); + } + + public static string BuildGlossaryMarkdown( + IReadOnlyDictionary autoPairs, + IReadOnlyDictionary manualPairs) + { + var sb = new StringBuilder(); + sb.AppendLine($"# {GlobalWikiCatalog.GlossaryTitle}"); + sb.AppendLine(); + sb.AppendLine( + $"_Updated: {DateTimeOffset.UtcNow:yyyy-MM-dd HH:mm} UTC · {autoPairs.Count} auto · {manualPairs.Count} manual_"); + sb.AppendLine(); + sb.AppendLine(AutoSectionHeader); + sb.AppendLine("_Regenerated from document digests on each `digests/rebuild`. Do not edit._"); + sb.AppendLine(); + foreach (var kv in autoPairs.OrderBy(k => k.Key, StringComparer.OrdinalIgnoreCase)) + sb.AppendLine($"- {FormatGlossaryEntry(kv.Key, kv.Value)}"); + if (autoPairs.Count == 0) + sb.AppendLine("_No acronym pairs harvested yet._"); + sb.AppendLine(); + sb.AppendLine(ManualSectionHeader); + sb.AppendLine("_Org-wide acronyms not visible in tickets. Edit here; preserved on rebuild._"); + sb.AppendLine(); + foreach (var kv in manualPairs.OrderBy(k => k.Key, StringComparer.OrdinalIgnoreCase)) + sb.AppendLine($"- {FormatGlossaryEntry(kv.Key, kv.Value)}"); + if (manualPairs.Count == 0) + sb.AppendLine("_None._"); + return sb.ToString().TrimEnd() + "\n"; + } + + /// Split an existing glossary into manual overrides and (ignored) auto section. + public static (Dictionary Manual, Dictionary Auto) ParseGlossarySections( + string? markdown) + { + var manual = new Dictionary(StringComparer.OrdinalIgnoreCase); + var auto = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (string.IsNullOrWhiteSpace(markdown)) + return (manual, auto); + + var hasStructuredSections = markdown.Contains(AutoSectionHeader, StringComparison.Ordinal) + || markdown.Contains(ManualSectionHeader, StringComparison.Ordinal); + + var section = hasStructuredSections ? GlossarySection.None : GlossarySection.Manual; + foreach (var raw in markdown.Replace("\r\n", "\n", StringComparison.Ordinal).Split('\n')) + { + var line = raw.Trim(); + if (line.Length == 0) + continue; + if (line.StartsWith('#')) + { + if (line.StartsWith(ManualSectionHeader, StringComparison.OrdinalIgnoreCase)) + section = GlossarySection.Manual; + else if (line.StartsWith(AutoSectionHeader, StringComparison.OrdinalIgnoreCase)) + section = GlossarySection.Auto; + else if (!hasStructuredSections) + continue; + else + continue; + continue; + } + + if (line.StartsWith('_') && line.EndsWith('_')) + continue; + if (line.StartsWith('|') && line.Contains("---", StringComparison.Ordinal)) + continue; + + var target = section switch + { + GlossarySection.Auto => auto, + GlossarySection.Manual => manual, + _ => manual + }; + TryParseGlossaryLine(line, target); + } + + return (manual, auto); + } + + private enum GlossarySection + { + None, + Auto, + Manual + } + + private static void TryParseGlossaryLine(string line, Dictionary target) + { + var scratch = new GlobalWikiAliasLexicon(); + scratch.ParseGlossaryLineInto(line, target); + } + + private void ParseGlossaryLineInto(string line, Dictionary target) + { + var match = GlossaryLine.Match(line); + if (match.Success) + { + AddToDictionary(match.Groups[1].Value, match.Groups[2].Value, target); + return; + } + + HarvestLineInto(line, target); + } + + private void AddToDictionary(string? acronymRaw, string? expansionRaw, Dictionary target) + { + var acronym = NormalizeAcronym(acronymRaw); + var expansion = NormalizeExpansion(expansionRaw); + if (acronym is null || expansion is null) + return; + if (string.Equals(acronym, expansion, StringComparison.OrdinalIgnoreCase)) + return; + target[acronym] = expansion; + } + + private static string FormatGlossaryEntry(string acronym, string expansion) + { + var particles = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "de", "da", "do", "dos", "das", "of", "the", "and", "e" + }; + var display = string.Join( + ' ', + expansion.Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Select((w, i) => + i > 0 && (particles.Contains(w) || w.Length <= 3) + ? w + : char.ToUpperInvariant(w[0]) + w[1..])); + return $"{acronym.ToUpperInvariant()}: {display}"; + } + + public static void Remember(string appId, GlobalWikiAliasLexicon lexicon) + { + if (string.IsNullOrWhiteSpace(appId)) + return; + LastByApp[appId] = lexicon; + } + + public static void Invalidate(string appId) + { + if (!string.IsNullOrWhiteSpace(appId)) + LastByApp.TryRemove(appId, out _); + } + + public static GlobalWikiAliasLexicon ForApp(string appId) => + LastByApp.TryGetValue(appId, out var lexicon) ? lexicon : Empty; + + public static GlobalWikiAliasLexicon FromDocuments(IEnumerable documents) + { + var lexicon = new GlobalWikiAliasLexicon(); + foreach (var doc in documents) + { + if (GlobalWikiCatalog.IsGlossaryDocument(doc.DocumentId)) + { + lexicon.ParseGlossary(doc.Content); + continue; + } + + lexicon.Harvest(doc.Title, doc.Summary); + if (GlobalWikiCatalog.IsCatalogDocument(doc.DocumentId)) + lexicon.Harvest(doc.Title, doc.Content); + } + + return lexicon; + } + + public static GlobalWikiAliasLexicon FromHarvest( + string? glossaryMarkdown, + IEnumerable<(string Title, string Summary)> rows) + { + var lexicon = new GlobalWikiAliasLexicon(); + lexicon.ParseGlossary(glossaryMarkdown); + foreach (var (title, summary) in rows) + lexicon.Harvest(title, summary); + return lexicon; + } + + public GlobalWikiQueryExpansion Expand(string? query) + { + var original = query ?? string.Empty; + var tokens = GlobalWikiScoring.Tokenize(original).ToList(); + if (tokens.Count == 0) + { + return new GlobalWikiQueryExpansion + { + OriginalQuery = original, + OriginalPhrase = string.Empty, + Groups = [] + }; + } + + var groups = new List(); + var consumed = new bool[tokens.Count]; + + var expansions = _expansionToAcronym.Keys + .Select(e => (Phrase: e, Parts: e.Split(' ', StringSplitOptions.RemoveEmptyEntries))) + .Where(e => e.Parts.Length >= 2) + .OrderByDescending(e => e.Parts.Length) + .ToList(); + + for (var i = 0; i < tokens.Count; i++) + { + if (consumed[i]) + continue; + + var matchedExpansion = false; + foreach (var (phrase, parts) in expansions) + { + if (i + parts.Length > tokens.Count) + continue; + if (consumed.Skip(i).Take(parts.Length).Any(c => c)) + continue; + if (!parts.SequenceEqual(tokens.Skip(i).Take(parts.Length), StringComparer.OrdinalIgnoreCase)) + continue; + + for (var j = 0; j < parts.Length; j++) + consumed[i + j] = true; + + var acronym = _expansionToAcronym[phrase]; + var fullPhrase = _acronymToExpansion.TryGetValue(acronym, out var mapped) + ? mapped + : phrase; + groups.Add(BuildGroup(fullPhrase, acronym, fullPhrase)); + matchedExpansion = true; + break; + } + + if (matchedExpansion) + continue; + + consumed[i] = true; + var token = tokens[i]; + if (_acronymToExpansion.TryGetValue(token, out var expansion)) + groups.Add(BuildGroup(token, token, expansion)); + else + groups.Add(BuildGroup(token, acronym: null, expansionPhrase: null)); + } + + return new GlobalWikiQueryExpansion + { + OriginalQuery = original, + OriginalPhrase = string.Join(' ', tokens), + Groups = groups + }; + } + + /// Sanitized simple tsquery: synonym ORs, AND across original groups. + public static string? ToPostgresTsQuery(GlobalWikiQueryExpansion expansion) + { + if (expansion.Groups.Count == 0) + return null; + + var groups = new List(); + foreach (var group in expansion.Groups) + { + var alts = new List(); + var acronym = SanitizeTsTerm(group.Acronym); + if (acronym is not null) + alts.Add(acronym); + + var andTerms = group.ExpansionIndexTokens + .Select(SanitizeTsTerm) + .Where(t => t is not null) + .Cast() + .ToList(); + if (andTerms.Count == 1) + alts.Add(andTerms[0]); + else if (andTerms.Count > 1) + alts.Add("(" + string.Join(" & ", andTerms) + ")"); + + if (alts.Count == 0) + { + var fallback = SanitizeTsTerm(group.Canonical); + if (fallback is not null) + alts.Add(fallback); + } + + if (alts.Count == 1) + groups.Add(alts[0]); + else if (alts.Count > 1) + groups.Add("(" + string.Join(" | ", alts) + ")"); + } + + return groups.Count == 0 ? null : string.Join(" & ", groups); + } + + public void ParseGlossary(string? markdown) + { + if (string.IsNullOrWhiteSpace(markdown)) + return; + + var (manual, auto) = ParseGlossarySections(markdown); + foreach (var kv in manual) + TryAdd(kv.Key, kv.Value); + foreach (var kv in auto) + TryAdd(kv.Key, kv.Value); + } + + public void Harvest(string? title, string? text) + { + HarvestChunk(title); + HarvestChunk(text); + } + + private void HarvestChunk(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + return; + + foreach (Match match in Parenthetical.Matches(text)) + TryAdd(match.Groups[1].Value, match.Groups[2].Value); + + foreach (Match match in EmDashPair.Matches(text.Replace("\r\n", "\n", StringComparison.Ordinal))) + TryAdd(match.Groups[1].Value, match.Groups[2].Value); + + foreach (Match match in AliasesLine.Matches(text.Replace("\r\n", "\n", StringComparison.Ordinal))) + ParseAliasAssignments(match.Groups[1].Value); + } + + private void HarvestLineInto(string line, Dictionary target) + { + foreach (Match match in Parenthetical.Matches(line)) + AddToDictionary(match.Groups[1].Value, match.Groups[2].Value, target); + + var emDash = EmDashPair.Match(line); + if (emDash.Success) + AddToDictionary(emDash.Groups[1].Value, emDash.Groups[2].Value, target); + } + + private void ParseAliasAssignments(string assignments) + { + foreach (var part in assignments.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var eq = part.IndexOf('='); + var colon = part.IndexOf(':'); + var sep = eq >= 0 && (colon < 0 || eq < colon) ? eq : colon; + if (sep <= 0 || sep >= part.Length - 1) + continue; + TryAdd(part[..sep].Trim(), part[(sep + 1)..].Trim()); + } + } + + private void TryAdd(string? acronymRaw, string? expansionRaw) + { + var acronym = NormalizeAcronym(acronymRaw); + var expansion = NormalizeExpansion(expansionRaw); + if (acronym is null || expansion is null) + return; + if (string.Equals(acronym, expansion, StringComparison.OrdinalIgnoreCase)) + return; + + _acronymToExpansion[acronym] = expansion; + _expansionToAcronym[expansion] = acronym; + var stripped = string.Join(' ', expansion.Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Where(p => !Stopwords.Contains(p))); + if (stripped.Length >= 6 && !string.Equals(stripped, expansion, StringComparison.Ordinal)) + _expansionToAcronym[stripped] = acronym; + } + + private static GlobalWikiSynonymGroup BuildGroup(string canonical, string? acronym, string? expansionPhrase) + { + var expansionTokens = string.IsNullOrWhiteSpace(expansionPhrase) + ? Array.Empty() + : expansionPhrase.Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Where(t => t.Length >= 2 && !Stopwords.Contains(t)) + .ToArray(); + + var index = new HashSet(StringComparer.OrdinalIgnoreCase) { canonical }; + if (!string.IsNullOrWhiteSpace(acronym)) + index.Add(acronym); + foreach (var token in expansionTokens) + index.Add(token); + + return new GlobalWikiSynonymGroup + { + Canonical = canonical, + Acronym = acronym?.ToLowerInvariant() ?? string.Empty, + ExpansionPhrase = expansionPhrase ?? string.Empty, + ExpansionIndexTokens = expansionTokens, + IndexTokens = index.ToList() + }; + } + + private static string? NormalizeAcronym(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + var trimmed = raw.Trim().Trim('[', ']', '`', '"', '\''); + if (trimmed.Length is < 2 or > 8) + return null; + if (!trimmed.All(char.IsLetterOrDigit)) + return null; + if (trimmed.Count(char.IsLetter) < 2) + return null; + if (BlockedAcronyms.Contains(trimmed)) + return null; + return trimmed.ToLowerInvariant(); + } + + private static string? NormalizeExpansion(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + var trimmed = raw.Trim().TrimEnd('.', ';', ',').Trim(); + if (trimmed.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + return null; + var parts = trimmed.Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2) + return null; + var joined = string.Join(' ', parts.Select(p => p.Trim().ToLowerInvariant())); + return joined.Length is >= 6 and <= 80 ? joined : null; + } + + private static string? SanitizeTsTerm(string? term) + { + if (string.IsNullOrWhiteSpace(term)) + return null; + var t = term.Trim().ToLowerInvariant(); + if (t.Length is < 2 or > 40) + return null; + return t.All(c => char.IsAsciiLetterOrDigit(c)) ? t : null; + } +} + +public sealed class GlobalWikiQueryExpansion +{ + public required string OriginalQuery { get; init; } + public required string OriginalPhrase { get; init; } + public required IReadOnlyList Groups { get; init; } + + public IEnumerable IndexTokens => + Groups.SelectMany(g => g.IndexTokens).Distinct(StringComparer.OrdinalIgnoreCase); +} + +public sealed class GlobalWikiSynonymGroup +{ + public required string Canonical { get; init; } + public required string Acronym { get; init; } + public required string ExpansionPhrase { get; init; } + public required IReadOnlyList ExpansionIndexTokens { get; init; } + public required IReadOnlyList IndexTokens { get; init; } + + public bool Hits(string haystack) + { + if (!string.IsNullOrEmpty(Acronym) + && haystack.Contains(Acronym, StringComparison.Ordinal)) + return true; + if (!string.IsNullOrEmpty(ExpansionPhrase) + && haystack.Contains(ExpansionPhrase, StringComparison.Ordinal)) + return true; + return haystack.Contains(Canonical, StringComparison.Ordinal); + } +} diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiCatalog.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiCatalog.cs index ff17407..b25ac23 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiCatalog.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiCatalog.cs @@ -5,7 +5,17 @@ public static class GlobalWikiCatalog public const string DocumentId = "wiki:catalog"; public const string Title = "Knowledge catalog"; + /// Tenant-authored acronym ↔ expansion pairs used by lexical search. + public const string GlossaryDocumentId = "wiki:glossary"; + public const string GlossaryTitle = "Acronym glossary"; + public static bool IsCatalogDocument(string? documentId) => !string.IsNullOrWhiteSpace(documentId) && documentId.StartsWith("wiki:catalog", StringComparison.OrdinalIgnoreCase); + + public static bool IsGlossaryDocument(string? documentId) => + string.Equals(documentId, GlossaryDocumentId, StringComparison.OrdinalIgnoreCase); + + public static bool IsReservedDocument(string? documentId) => + IsCatalogDocument(documentId) || IsGlossaryDocument(documentId); } diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs index f38cba3..36ce47d 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs @@ -25,7 +25,7 @@ Task GenerateAsync( /// public sealed class GlobalWikiDigestGenerator : IGlobalWikiDigestGenerator { - public const int MaxLines = 6; + public const int MaxLines = 7; public const int MaxChars = 2_000; private readonly ILlmAdapterResolver _adapterResolver; diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs index cc6c2bb..ca81418 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs @@ -8,42 +8,41 @@ public static class GlobalWikiScoring { public static IEnumerable<(GlobalWikiDocument Document, double Score)> ScoreMatches( IReadOnlyList docs, - string query) + string query, + GlobalWikiAliasLexicon? lexicon = null) { - var tokens = Tokenize(query); - var phrase = string.Join(' ', tokens); + var expansion = (lexicon ?? GlobalWikiAliasLexicon.Empty).Expand(query); return docs - .Select(d => (Document: d, Score: ScoreDocument(d, tokens, phrase))) - .Where(x => tokens.Count == 0 || x.Score > 0) + .Select(d => (Document: d, Score: ScoreDocument(d, expansion))) + .Where(x => expansion.Groups.Count == 0 || x.Score > 0) .OrderByDescending(x => x.Score) .ThenBy(x => x.Document.Content.Length); } - public static double ScoreDocument(GlobalWikiDocument doc, HashSet tokens, string phrase) + public static double ScoreDocument(GlobalWikiDocument doc, GlobalWikiQueryExpansion expansion) { var score = 0.0; - if (tokens.Count > 0) + if (expansion.Groups.Count > 0) { var identity = $"{doc.DocumentId} {doc.Slug} {doc.Title}".ToLowerInvariant(); var summary = (doc.Summary ?? string.Empty).ToLowerInvariant(); var body = $"{doc.Summary} {doc.Content} {doc.SourceId}".ToLowerInvariant(); - // Phrase boost when the full query appears. - if (phrase.Length >= 4 && body.Contains(phrase, StringComparison.Ordinal)) + if (expansion.OriginalPhrase.Length >= 4 + && body.Contains(expansion.OriginalPhrase, StringComparison.Ordinal)) score += 40; - foreach (var token in tokens) + foreach (var group in expansion.Groups) { - if (identity.Contains(token, StringComparison.Ordinal)) + if (group.Hits(identity)) score += 100; - else if (summary.Contains(token, StringComparison.Ordinal)) + else if (group.Hits(summary)) { - // Digest keywords / summary hits outrank generic body noise. score += summary.Contains("keywords:", StringComparison.Ordinal) ? 35 : 20; } - else if (body.Contains(token, StringComparison.Ordinal)) + else if (group.Hits(body)) score += 10; } @@ -56,6 +55,23 @@ public static double ScoreDocument(GlobalWikiDocument doc, HashSet token return score; } + public static double ScoreDocument(GlobalWikiDocument doc, HashSet tokens, string phrase) => + ScoreDocument( + doc, + new GlobalWikiQueryExpansion + { + OriginalQuery = phrase, + OriginalPhrase = phrase, + Groups = tokens.Select(t => new GlobalWikiSynonymGroup + { + Canonical = t, + Acronym = string.Empty, + ExpansionPhrase = string.Empty, + ExpansionIndexTokens = [], + IndexTokens = [t] + }).ToList() + }); + public static HashSet Tokenize(string? text) { if (string.IsNullOrWhiteSpace(text)) diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs index c9f050e..ab6fc77 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs @@ -48,7 +48,7 @@ public async Task UpsertBatchAsync( { if (string.IsNullOrWhiteSpace(doc.DocumentId) || string.IsNullOrWhiteSpace(doc.Content)) continue; - if (GlobalWikiCatalog.IsCatalogDocument(doc.DocumentId)) + if (GlobalWikiCatalog.IsReservedDocument(doc.DocumentId)) continue; var result = await _store.UpsertAsync( @@ -87,7 +87,7 @@ public async Task RebuildDigestsAsync( .ConfigureAwait(false); var candidates = docs - .Where(d => !GlobalWikiCatalog.IsCatalogDocument(d.DocumentId)) + .Where(d => !GlobalWikiCatalog.IsReservedDocument(d.DocumentId)) .OrderBy(d => d.DocumentId, StringComparer.OrdinalIgnoreCase) .ToList(); @@ -143,13 +143,15 @@ await _store.UpsertAsync( } await RefreshCatalogAsync(appId, cancellationToken).ConfigureAwait(false); + var glossary = await RefreshGlossaryAsync(appId, cancellationToken).ConfigureAwait(false); _logger.LogInformation( - "Wiki digests rebuilt for {AppId}: processed={Processed}, updated={Updated}, skipped={Skipped}", + "Wiki digests rebuilt for {AppId}: processed={Processed}, updated={Updated}, skipped={Skipped}, glossaryPairs={GlossaryPairs}", appId, candidates.Count, updated, - skipped); + skipped, + glossary.PairCount); return new GlobalWikiDigestRebuildResult { @@ -157,18 +159,23 @@ await _store.UpsertAsync( Processed = candidates.Count, Updated = updated, Skipped = skipped, - CatalogRefreshed = true + CatalogRefreshed = true, + GlossaryRefreshed = glossary.Refreshed, + GlossaryPairs = glossary.PairCount }; } public async Task DeleteAsync(string appId, string documentId, CancellationToken cancellationToken = default) { - if (GlobalWikiCatalog.IsCatalogDocument(documentId)) + if (GlobalWikiCatalog.IsReservedDocument(documentId)) return await _store.DeleteAsync(appId, documentId, cancellationToken).ConfigureAwait(false); var deleted = await _store.DeleteAsync(appId, documentId, cancellationToken).ConfigureAwait(false); if (deleted) + { await RefreshCatalogAsync(appId, cancellationToken).ConfigureAwait(false); + await RefreshGlossaryAsync(appId, cancellationToken).ConfigureAwait(false); + } return deleted; } @@ -220,6 +227,7 @@ public async Task QueryAsync( var matchedDocs = (await _store .SearchAsync(appId, request.Query, asOf, request.SourceId, topK, cancellationToken) .ConfigureAwait(false)) + .Where(d => !GlobalWikiCatalog.IsGlossaryDocument(d.DocumentId)) .ToList(); var totalDocs = await _store @@ -241,7 +249,8 @@ public async Task QueryAsync( // Prefer scored ordering from GlobalWikiScoring when we have the pool. if (matchedDocs.Count > 0) { - var rescored = GlobalWikiScoring.ScoreMatches(matchedDocs, request.Query).ToList(); + var lexicon = GlobalWikiAliasLexicon.ForApp(appId); + var rescored = GlobalWikiScoring.ScoreMatches(matchedDocs, request.Query, lexicon).ToList(); matchedDocs = rescored.Select(x => x.Document).ToList(); matches = rescored .Select(m => new GlobalWikiMatch @@ -493,7 +502,7 @@ private async Task RefreshCatalogAsync(string appId, CancellationToken cancellat var docs = await _store.GetAllForQueryAsync(appId, sourceId: null, asOf: null, cancellationToken) .ConfigureAwait(false); var entries = docs - .Where(d => !GlobalWikiCatalog.IsCatalogDocument(d.DocumentId)) + .Where(d => !GlobalWikiCatalog.IsReservedDocument(d.DocumentId)) .OrderBy(d => d.DocumentId, StringComparer.OrdinalIgnoreCase) .ToList(); @@ -540,6 +549,64 @@ await _store.UpsertAsync( } } + private async Task<(bool Refreshed, int PairCount)> RefreshGlossaryAsync( + string appId, + CancellationToken cancellationToken) + { + try + { + var docs = await _store + .GetAllForQueryAsync(appId, sourceId: null, asOf: null, cancellationToken) + .ConfigureAwait(false); + + var existing = await _store + .GetAsync(appId, GlobalWikiCatalog.GlossaryDocumentId, cancellationToken) + .ConfigureAwait(false); + var (manual, _) = GlobalWikiAliasLexicon.ParseGlossarySections(existing?.Content); + + var harvested = GlobalWikiAliasLexicon.HarvestPairsFromDocuments(docs); + var auto = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var kv in harvested) + { + if (manual.ContainsKey(kv.Key)) + continue; + auto[kv.Key] = kv.Value; + } + + var totalPairs = auto.Count + manual.Count; + if (totalPairs == 0 && existing is null) + return (Refreshed: false, PairCount: 0); + + var markdown = GlobalWikiAliasLexicon.BuildGlossaryMarkdown(auto, manual); + var unchanged = existing is not null + && string.Equals(existing.Content.Trim(), markdown.Trim(), StringComparison.Ordinal); + if (unchanged) + return (Refreshed: true, PairCount: totalPairs); + + await _store.UpsertAsync( + appId, + GlobalWikiCatalog.GlossaryDocumentId, + new GlobalWikiUpsertRequest + { + Title = GlobalWikiCatalog.GlossaryTitle, + Content = markdown, + Summary = $"Acronym glossary: {auto.Count} auto from digests, {manual.Count} manual.", + SourceId = "wiki:glossary", + Slug = "wiki-glossary", + Overwrite = true + }, + cancellationToken).ConfigureAwait(false); + + GlobalWikiAliasLexicon.Invalidate(appId); + return (Refreshed: true, PairCount: totalPairs); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to refresh global wiki glossary for {AppId}", appId); + return (Refreshed: false, PairCount: 0); + } + } + private static GlobalWikiDocumentSummary ToSummary(GlobalWikiDocument d) => new() { diff --git a/src/ContextMemory.Core/Localization/LlmPrompts.cs b/src/ContextMemory.Core/Localization/LlmPrompts.cs index 59cc80c..eddcea7 100644 --- a/src/ContextMemory.Core/Localization/LlmPrompts.cs +++ b/src/ContextMemory.Core/Localization/LlmPrompts.cs @@ -234,9 +234,11 @@ public static string WikiTicketDigest(string? language) => FULL CONTENT (includes comments — extract rules/policies from them): {content} - Write at most 6 lines of plain text (no markdown fences, no JSON): + Write at most 7 lines of plain text (no markdown fences, no JSON): - Line 1 MUST be: Keywords: k1, k2, k3, ... (5–12 specific terms: IDs, systems, rules, entities) - - Lines 2–6: short factual bullets covering problem, decisions, and especially RULES found in comments/discussion + - When the document uses an acronym, include BOTH forms in Keywords, e.g. VVE (Virtual Vehicle Enablement) + - Line 2 (only when acronyms appear in the document): Aliases: ACRO=Full expansion; ACRO2=Other expansion + - Lines 3–7: short factual bullets covering problem, decisions, and especially RULES found in comments/discussion - Prefer concrete constraints ("must", "cannot", SLAs, formulas, owners) over vague summary - Same language as the document - Do not invent facts; omit unknowns @@ -251,9 +253,11 @@ Resume este documento da base de conhecimento para pesquisa. CONTEÚDO COMPLETO (inclui comentários — extrai regras/políticas deles): {content} - Escreve no máximo 6 linhas de texto simples (sem fences markdown, sem JSON): + Escreve no máximo 7 linhas de texto simples (sem fences markdown, sem JSON): - A linha 1 DEVE ser: Keywords: k1, k2, k3, ... (5–12 termos específicos: IDs, sistemas, regras, entidades) - - Linhas 2–6: factos curtos sobre problema, decisões e sobretudo REGRAS nos comentários/discussão + - Quando o documento usa uma sigla, inclui as DUAS formas em Keywords, p.ex. VVE (Virtual Vehicle Enablement) + - Linha 2 (só se houver siglas no documento): Aliases: SIGLA=Expansão completa; SIGLA2=Outra expansão + - Linhas 3–7: factos curtos sobre problema, decisões e sobretudo REGRAS nos comentários/discussão - Prefere restrições concretas ("deve", "não pode", SLAs, fórmulas, owners) a resumos vagos - Mesma língua do documento - Não inventes factos; omite o que não souberes diff --git a/src/ContextMemory.Core/Models/GlobalWikiModels.cs b/src/ContextMemory.Core/Models/GlobalWikiModels.cs index f576e8c..af85487 100644 --- a/src/ContextMemory.Core/Models/GlobalWikiModels.cs +++ b/src/ContextMemory.Core/Models/GlobalWikiModels.cs @@ -130,6 +130,12 @@ public sealed class GlobalWikiDigestRebuildResult [JsonPropertyName("catalogRefreshed")] public bool CatalogRefreshed { get; init; } + + [JsonPropertyName("glossaryRefreshed")] + public bool GlossaryRefreshed { get; init; } + + [JsonPropertyName("glossaryPairs")] + public int GlossaryPairs { get; init; } } public sealed class GlobalWikiBatchDocument diff --git a/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs b/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs index e440bf3..b161748 100644 --- a/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs +++ b/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs @@ -82,6 +82,7 @@ public async Task UpsertAsync( CancellationToken cancellationToken = default) { await using var db = await _dbFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + GlobalWikiAliasLexicon.Invalidate(appId); var hash = GlobalWikiSlug.ComputeContentHash(request.Content); var existing = await db.GlobalWikiDocuments .FirstOrDefaultAsync( @@ -248,6 +249,7 @@ public async Task DeleteAsync( existing.ValidTo = now; existing.UpdatedAt = now; await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + GlobalWikiAliasLexicon.Invalidate(appId); return true; } @@ -290,31 +292,44 @@ public async Task> SearchAsync( var point = asOf ?? DateTimeOffset.UtcNow; await using var db = await _dbFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var lexicon = await LoadLexiconAsync(db, appId, point, cancellationToken).ConfigureAwait(false); + GlobalWikiAliasLexicon.Remember(appId, lexicon); + var expansion = lexicon.Expand(query); + var tsQuery = GlobalWikiAliasLexicon.ToPostgresTsQuery(expansion); + List entities; try { - var tokens = GlobalWikiScoring.Tokenize(query); - if (tokens.Count == 0) + if (expansion.Groups.Count == 0) { return (await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false)) .Take(topK) .ToList(); } - // Use GIN-backed search_vector when present; plainto_tsquery is safer for user input. + if (string.IsNullOrWhiteSpace(tsQuery)) + { + var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); + return GlobalWikiScoring.ScoreMatches(docs, query, lexicon) + .Take(topK) + .Select(x => x.Document) + .ToList(); + } + + // Synonym groups as OR, AND across original query terms. Safer than raw user plainto_tsquery. var sql = """ SELECT * FROM global_wiki_documents WHERE "AppId" = {0} AND "ValidFrom" <= {1} AND ("ValidTo" IS NULL OR "ValidTo" > {1}) AND ({2}::text IS NULL OR "SourceId" = {2}) - AND search_vector @@ plainto_tsquery('simple', {3}) - ORDER BY ts_rank(search_vector, plainto_tsquery('simple', {3})) DESC + AND search_vector @@ to_tsquery('simple', {3}) + ORDER BY ts_rank(search_vector, to_tsquery('simple', {3})) DESC LIMIT {4} """; entities = await db.GlobalWikiDocuments - .FromSqlRaw(sql, appId, point, (object?)sourceId ?? DBNull.Value, query, topK) + .FromSqlRaw(sql, appId, point, (object?)sourceId ?? DBNull.Value, tsQuery, topK) .AsNoTracking() .ToListAsync(cancellationToken) .ConfigureAwait(false); @@ -322,7 +337,7 @@ ORDER BY ts_rank(search_vector, plainto_tsquery('simple', {3})) DESC catch { var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); - return GlobalWikiScoring.ScoreMatches(docs, query) + return GlobalWikiScoring.ScoreMatches(docs, query, lexicon) .Take(topK) .Select(x => x.Document) .ToList(); @@ -331,18 +346,57 @@ ORDER BY ts_rank(search_vector, plainto_tsquery('simple', {3})) DESC if (entities.Count == 0) { var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); - return GlobalWikiScoring.ScoreMatches(docs, query) + return GlobalWikiScoring.ScoreMatches(docs, query, lexicon) .Take(topK) .Select(x => x.Document) .ToList(); } - return GlobalWikiScoring.ScoreMatches(entities.Select(ToDocument).ToList(), query) + return GlobalWikiScoring.ScoreMatches(entities.Select(ToDocument).ToList(), query, lexicon) .Take(topK) .Select(x => x.Document) .ToList(); } + private static async Task LoadLexiconAsync( + ContextMemoryDbContext db, + string appId, + DateTimeOffset point, + CancellationToken cancellationToken) + { + var rows = await db.GlobalWikiDocuments.AsNoTracking() + .Where(x => x.AppId == appId + && x.ValidFrom <= point + && (x.ValidTo == null || x.ValidTo > point)) + .Select(x => new { x.DocumentId, x.Title, x.Summary }) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var reserved = await db.GlobalWikiDocuments.AsNoTracking() + .Where(x => x.AppId == appId + && x.ValidFrom <= point + && (x.ValidTo == null || x.ValidTo > point) + && (x.DocumentId == GlobalWikiCatalog.GlossaryDocumentId + || x.DocumentId == GlobalWikiCatalog.DocumentId)) + .Select(x => new { x.DocumentId, x.Content }) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + string? glossary = reserved + .FirstOrDefault(r => GlobalWikiCatalog.IsGlossaryDocument(r.DocumentId)) + ?.Content; + var harvest = rows + .Select(r => (r.Title, r.Summary)) + .ToList(); + var catalog = reserved + .FirstOrDefault(r => GlobalWikiCatalog.IsCatalogDocument(r.DocumentId)) + ?.Content; + if (!string.IsNullOrWhiteSpace(catalog)) + harvest.Add((GlobalWikiCatalog.Title, catalog)); + + return GlobalWikiAliasLexicon.FromHarvest(glossary, harvest); + } + public async Task> ListRevisionsAsync( string appId, string documentId, diff --git a/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs b/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs index 2005ca7..973319a 100644 --- a/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs +++ b/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs @@ -75,6 +75,7 @@ public async Task UpsertAsync( await gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { + GlobalWikiAliasLexicon.Invalidate(appId); EnsureDirs(appId, documentId); var hash = GlobalWikiSlug.ComputeContentHash(request.Content); @@ -274,6 +275,7 @@ public async Task DeleteAsync( if (File.Exists(pointer)) File.Delete(pointer); await RebuildIndexAsync(appId, cancellationToken).ConfigureAwait(false); + GlobalWikiAliasLexicon.Invalidate(appId); return true; } @@ -303,7 +305,10 @@ public async Task> SearchAsync( { var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); var index = await ReadIndexAsync(appId, cancellationToken).ConfigureAwait(false); - var tokens = Tokenize(query); + var lexicon = GlobalWikiAliasLexicon.FromDocuments(docs); + GlobalWikiAliasLexicon.Remember(appId, lexicon); + var expansion = lexicon.Expand(query); + var tokens = expansion.IndexTokens.ToHashSet(StringComparer.OrdinalIgnoreCase); if (tokens.Count == 0) return docs.Take(Math.Max(1, topK)).ToList(); @@ -326,7 +331,7 @@ public async Task> SearchAsync( ? docs.Where(d => candidateIds.Contains(d.DocumentId)).ToList() : docs; - return GlobalWikiScoring.ScoreMatches(pool, query) + return GlobalWikiScoring.ScoreMatches(pool, query, lexicon) .Take(Math.Clamp(topK, 1, 200)) .Select(x => x.Document) .ToList(); diff --git a/tests/ContextMemory.Api.Tests/GlobalWikiAliasLexiconTests.cs b/tests/ContextMemory.Api.Tests/GlobalWikiAliasLexiconTests.cs new file mode 100644 index 0000000..f7859fd --- /dev/null +++ b/tests/ContextMemory.Api.Tests/GlobalWikiAliasLexiconTests.cs @@ -0,0 +1,142 @@ +using ContextMemory.Core.GlobalWiki; +using ContextMemory.Core.Models; +using Xunit; + +namespace ContextMemory.Api.Tests; + +public sealed class GlobalWikiAliasLexiconTests +{ + [Fact] + public void Glossary_ExpandsAcronymAndPhrase_Bidirectionally() + { + var lexicon = new GlobalWikiAliasLexicon(); + lexicon.ParseGlossary( + """ + # Glossary + - RJ: Rio de Janeiro + - VVE: Virtual Vehicle Enablement + """); + + Assert.Equal(2, lexicon.PairCount); + + var vve = lexicon.Expand("VVE"); + Assert.Contains(vve.Groups, g => + g.Acronym == "vve" && g.ExpansionPhrase == "virtual vehicle enablement"); + + var rio = lexicon.Expand("Rio de Janeiro"); + Assert.Contains(rio.Groups, g => + g.Acronym == "rj" && g.ExpansionPhrase == "rio de janeiro"); + + var ts = GlobalWikiAliasLexicon.ToPostgresTsQuery(vve); + Assert.NotNull(ts); + Assert.Contains("vve", ts, StringComparison.Ordinal); + Assert.Contains("virtual", ts, StringComparison.Ordinal); + Assert.Contains("&", ts, StringComparison.Ordinal); + } + + [Fact] + public void ParseGlossarySections_PreservesManualOverrides() + { + var markdown = GlobalWikiAliasLexicon.BuildGlossaryMarkdown( + new Dictionary(StringComparer.OrdinalIgnoreCase) { ["vve"] = "virtual vehicle enablement" }, + new Dictionary(StringComparer.OrdinalIgnoreCase) { ["rj"] = "rio de janeiro" }); + + var (manual, auto) = GlobalWikiAliasLexicon.ParseGlossarySections(markdown); + Assert.Equal("rio de janeiro", manual["rj"]); + Assert.Equal("virtual vehicle enablement", auto["vve"]); + } + + [Fact] + public void Harvest_ReadsAliasesLine() + { + var lexicon = GlobalWikiAliasLexicon.FromHarvest( + glossaryMarkdown: null, + [ + ("Fleet", """ + Keywords: VVE, fleet + Aliases: VVE=Virtual Vehicle Enablement + Rule: enable before SOP. + """) + ]); + + Assert.Contains(lexicon.Expand("VVE").Groups, g => g.ExpansionPhrase == "virtual vehicle enablement"); + } + + [Fact] + public void BuildGlossaryMarkdown_FormatsEntries() + { + var md = GlobalWikiAliasLexicon.BuildGlossaryMarkdown( + new Dictionary(StringComparer.OrdinalIgnoreCase) { ["vve"] = "virtual vehicle enablement" }, + new Dictionary(StringComparer.OrdinalIgnoreCase)); + Assert.Contains("## Auto", md); + Assert.Contains("VVE: Virtual Vehicle Enablement", md); + } + + [Fact] + public void Harvest_ReadsParentheticalAndEmDashPairs() + { + var lexicon = GlobalWikiAliasLexicon.FromHarvest( + glossaryMarkdown: null, + [ + ("VVE — Virtual Vehicle Enablement", "Keywords: VVE (Virtual Vehicle Enablement), fleet") + ]); + + var expansion = lexicon.Expand("vve"); + Assert.Contains(expansion.Groups, g => g.ExpansionPhrase == "virtual vehicle enablement"); + } + + [Fact] + public void ScoreMatches_FindsExpansionWhenQueryIsAcronym() + { + var lexicon = new GlobalWikiAliasLexicon(); + lexicon.ParseGlossary("- VVE: Virtual Vehicle Enablement"); + + var hit = Doc("vve-doc", "Fleet spec", "Virtual Vehicle Enablement is required for EU markets."); + var decoy = Doc("other", "Parking", "Vehicle parking policy for the office."); + + var ranked = GlobalWikiScoring.ScoreMatches([decoy, hit], "VVE", lexicon).Select(x => x.Document.DocumentId).ToList(); + + Assert.Equal("vve-doc", ranked[0]); + Assert.Contains("vve-doc", ranked); + } + + [Fact] + public void ScoreMatches_FindsAcronymWhenQueryIsExpansion() + { + var lexicon = new GlobalWikiAliasLexicon(); + lexicon.ParseGlossary("- RJ: Rio de Janeiro"); + + var hit = Doc("rj-site", "Plant RJ", "The RJ plant handles south-east logistics."); + var decoy = Doc("other", "Lisbon", "Lisbon office hours."); + + var ranked = GlobalWikiScoring.ScoreMatches([decoy, hit], "Rio de Janeiro", lexicon) + .Select(x => x.Document.DocumentId) + .ToList(); + + Assert.Equal("rj-site", ranked[0]); + } + + [Fact] + public void ScoreMatches_DoesNotTreatLooseWordAsAcronymHit() + { + var lexicon = new GlobalWikiAliasLexicon(); + lexicon.ParseGlossary("- VVE: Virtual Vehicle Enablement"); + + var decoy = Doc("parking", "Parking", "Vehicle parking only."); + var ranked = GlobalWikiScoring.ScoreMatches([decoy], "VVE", lexicon).ToList(); + + Assert.DoesNotContain(ranked, x => x.Score >= 100); + } + + private static GlobalWikiDocument Doc(string id, string title, string content) => + new() + { + AppId = "demo", + DocumentId = id, + Slug = id, + Title = title, + Content = content, + Summary = string.Empty, + UpdatedAt = DateTimeOffset.UtcNow + }; +} diff --git a/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs b/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs index 8f83a2c..05d2f20 100644 --- a/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs +++ b/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs @@ -178,6 +178,153 @@ public async Task Query_DigestOnly_PacksSummaryNotFullBody() } } + [Fact] + public async Task RebuildDigests_AutoRefreshesGlossaryFromSummaries() + { + var root = Path.Combine(Path.GetTempPath(), "cm-global-wiki-" + Guid.NewGuid().ToString("N")); + try + { + var digest = new StubDigestGenerator + { + Digest = + """ + Keywords: VVE, fleet, EU + Aliases: VVE=Virtual Vehicle Enablement + Virtual Vehicle Enablement is mandatory before SOP. + """ + }; + var service = CreateService(root, digest); + + await service.UpsertAsync("demo", "fleet-eu", new GlobalWikiUpsertRequest + { + Title = "EU fleet rollout", + Content = "# EU fleet\n\nVirtual Vehicle Enablement is mandatory before SOP.", + SourceId = "confluence:FLEET" + }); + + var rebuild = await service.RebuildDigestsAsync("demo", new GlobalWikiDigestRebuildRequest()); + Assert.True(rebuild.GlossaryRefreshed); + Assert.True(rebuild.GlossaryPairs >= 1); + + var glossary = await CreateStore(root).GetAsync("demo", GlobalWikiCatalog.GlossaryDocumentId); + Assert.NotNull(glossary); + Assert.Contains("VVE", glossary!.Content, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Virtual Vehicle Enablement", glossary.Content, StringComparison.OrdinalIgnoreCase); + Assert.Contains("## Auto", glossary.Content); + + var byAcronym = await service.QueryAsync("demo", new GlobalWikiQueryRequest + { + Query = "VVE", + TopK = 5, + BudgetChars = 4000 + }); + Assert.Contains(byAcronym.Matches, m => m.DocumentId == "fleet-eu"); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public async Task RebuildDigests_PreservesManualGlossaryOverrides() + { + var root = Path.Combine(Path.GetTempPath(), "cm-global-wiki-" + Guid.NewGuid().ToString("N")); + try + { + var service = CreateService(root); + await service.UpsertAsync("demo", GlobalWikiCatalog.GlossaryDocumentId, new GlobalWikiUpsertRequest + { + Title = GlobalWikiCatalog.GlossaryTitle, + Content = GlobalWikiAliasLexicon.BuildGlossaryMarkdown( + new Dictionary(StringComparer.OrdinalIgnoreCase), + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["rj"] = "rio de janeiro" + }), + SourceId = "wiki:glossary", + Overwrite = true + }); + + var digest = new StubDigestGenerator + { + Digest = "Keywords: VVE (Virtual Vehicle Enablement)\nRule line." + }; + service = new GlobalWikiService(CreateStore(root), digest, NullLogger.Instance); + await service.UpsertAsync("demo", "fleet-eu", new GlobalWikiUpsertRequest + { + Title = "Fleet", + Content = "# Fleet\n\nVVE rollout.", + SourceId = "confluence:FLEET" + }); + + await service.RebuildDigestsAsync("demo", new GlobalWikiDigestRebuildRequest()); + + var glossary = await CreateStore(root).GetAsync("demo", GlobalWikiCatalog.GlossaryDocumentId); + Assert.NotNull(glossary); + Assert.Contains("RJ: Rio de Janeiro", glossary!.Content); + Assert.Contains("## Manual overrides", glossary.Content); + Assert.Contains("VVE", glossary.Content, StringComparison.OrdinalIgnoreCase); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public async Task Query_ResolvesAcronymsViaGlossary() + { + var root = Path.Combine(Path.GetTempPath(), "cm-global-wiki-" + Guid.NewGuid().ToString("N")); + try + { + var service = CreateService(root); + + await service.UpsertAsync("demo", GlobalWikiCatalog.GlossaryDocumentId, new GlobalWikiUpsertRequest + { + Title = GlobalWikiCatalog.GlossaryTitle, + Content = "- VVE: Virtual Vehicle Enablement\n- RJ: Rio de Janeiro\n", + SourceId = "wiki:glossary" + }); + await service.UpsertAsync("demo", "fleet-eu", new GlobalWikiUpsertRequest + { + Title = "EU fleet rollout", + Content = "# EU fleet\n\nVirtual Vehicle Enablement is mandatory before SOP.", + SourceId = "confluence:FLEET" + }); + await service.UpsertAsync("demo", "plant-rj", new GlobalWikiUpsertRequest + { + Title = "Plant RJ capacity", + Content = "# Plant RJ\n\nRJ line 2 is the overflow plant.", + SourceId = "confluence:PLANTS" + }); + + var byAcronym = await service.QueryAsync("demo", new GlobalWikiQueryRequest + { + Query = "VVE", + TopK = 5, + BudgetChars = 4000 + }); + Assert.Contains(byAcronym.Matches, m => m.DocumentId == "fleet-eu"); + Assert.DoesNotContain(byAcronym.Matches, m => m.DocumentId == GlobalWikiCatalog.GlossaryDocumentId); + + var byPlace = await service.QueryAsync("demo", new GlobalWikiQueryRequest + { + Query = "Rio de Janeiro", + TopK = 5, + BudgetChars = 4000 + }); + Assert.Contains(byPlace.Matches, m => m.DocumentId == "plant-rj"); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + [Fact] public async Task Query_PacksMatchedDocumentBody_BeforeFillerIndexPages() { @@ -232,7 +379,7 @@ public async Task Query_PacksMatchedDocumentBody_BeforeFillerIndexPages() } [Fact] - public void NormalizeDigest_EnforcesKeywordsLine_AndMaxSixLines() + public void NormalizeDigest_EnforcesKeywordsLine_AndMaxSevenLines() { var raw = """ @@ -242,15 +389,16 @@ Extra 1 Extra 2 Extra 3 Extra 4 - Extra 5 should drop + Extra 5 + Extra 6 should drop """; var normalized = GlobalWikiDigestGenerator.NormalizeDigest(raw, "PAC-668", "Billing"); var lines = normalized.Split('\n'); - Assert.Equal(6, lines.Length); + Assert.Equal(7, lines.Length); Assert.StartsWith("Keywords:", lines[0]); Assert.Contains("never reopen closed batches", normalized); - Assert.DoesNotContain("Extra 5 should drop", normalized); + Assert.DoesNotContain("Extra 6 should drop", normalized); } [Fact] From 33e3a26753612e36abcf80064d229a06883132d6 Mon Sep 17 00:00:00 2001 From: Vitor Castro Date: Mon, 17 Aug 2026 23:27:12 +0100 Subject: [PATCH 2/2] Improve digest-only wiki search with dedicated FTS index and question matching. Add digest_search_vector, OR tsquery fallback, Questions digest line, and remove full-corpus scan fallbacks so lazy retrieval stays fast and body-only terms do not leak in digest mode. --- .../Contracts/IGlobalWikiStore.cs | 1 + .../GlobalWiki/GlobalWikiAliasLexicon.cs | 12 ++ .../GlobalWiki/GlobalWikiDigestFields.cs | 38 ++++++ .../GlobalWiki/GlobalWikiDigestGenerator.cs | 4 +- .../GlobalWiki/GlobalWikiScoring.cs | 57 +++++++-- .../GlobalWiki/GlobalWikiService.cs | 15 ++- .../Localization/LlmPrompts.cs | 10 +- ...7120000_AddGlobalWikiDigestSearchVector.cs | 42 +++++++ .../Postgres/PostgresGlobalWikiStore.cs | 111 +++++++++++------- .../Wiki/FileGlobalWikiStore.cs | 69 +++++++---- .../GlobalWikiDigestSearchTests.cs | 70 +++++++++++ .../GlobalWikiServiceTests.cs | 42 ++++++- 12 files changed, 385 insertions(+), 86 deletions(-) create mode 100644 src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestFields.cs create mode 100644 src/ContextMemory.Infrastructure/Migrations/20260817120000_AddGlobalWikiDigestSearchVector.cs create mode 100644 tests/ContextMemory.Api.Tests/GlobalWikiDigestSearchTests.cs diff --git a/src/ContextMemory.Core/Contracts/IGlobalWikiStore.cs b/src/ContextMemory.Core/Contracts/IGlobalWikiStore.cs index 3e00491..1b9b1e6 100644 --- a/src/ContextMemory.Core/Contracts/IGlobalWikiStore.cs +++ b/src/ContextMemory.Core/Contracts/IGlobalWikiStore.cs @@ -47,6 +47,7 @@ Task> SearchAsync( DateTimeOffset? asOf = null, string? sourceId = null, int topK = 50, + bool digestOnly = false, CancellationToken cancellationToken = default); Task> ListRevisionsAsync( diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs index 71acc45..52c98bc 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiAliasLexicon.cs @@ -352,6 +352,18 @@ public GlobalWikiQueryExpansion Expand(string? query) return groups.Count == 0 ? null : string.Join(" & ", groups); } + /// Permissive OR tsquery — used when strict AND matching returns no hits. + public static string? ToPostgresOrTsQuery(GlobalWikiQueryExpansion expansion) + { + var terms = expansion.IndexTokens + .Select(SanitizeTsTerm) + .Where(t => t is not null) + .Cast() + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + return terms.Count == 0 ? null : string.Join(" | ", terms); + } + public void ParseGlossary(string? markdown) { if (string.IsNullOrWhiteSpace(markdown)) diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestFields.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestFields.cs new file mode 100644 index 0000000..dafb6b9 --- /dev/null +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestFields.cs @@ -0,0 +1,38 @@ +using System.Text.RegularExpressions; +using ContextMemory.Core.Models; + +namespace ContextMemory.Core.GlobalWiki; + +/// Parsed structured fields from LLM wiki digests (Summary). +public static class GlobalWikiDigestFields +{ + private static readonly Regex FieldLine = new( + @"^(Keywords|Aliases|Questions):\s*(.+)$", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + + public static IReadOnlyList ExtractQuestionPhrases(string? summary) + { + if (string.IsNullOrWhiteSpace(summary)) + return []; + + var phrases = new List(); + foreach (var raw in summary.Replace("\r\n", "\n", StringComparison.Ordinal).Split('\n')) + { + var match = FieldLine.Match(raw.Trim()); + if (!match.Success || !match.Groups[1].Value.Equals("Questions", StringComparison.OrdinalIgnoreCase)) + continue; + + foreach (var part in match.Groups[2].Value.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var normalized = GlobalWikiScoring.Tokenize(part); + if (normalized.Count >= 2) + phrases.Add(string.Join(' ', normalized.OrderBy(t => t, StringComparer.Ordinal))); + } + } + + return phrases; + } + + public static string DigestIndexText(GlobalWikiDocument doc) => + $"{doc.DocumentId} {doc.Slug} {doc.Title} {doc.Summary} {doc.SourceId}"; +} diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs index 36ce47d..8227bf3 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiDigestGenerator.cs @@ -21,11 +21,11 @@ Task GenerateAsync( } /// -/// Uses the summary LLM () to build a keyword + ≤6-line digest. +/// Uses the summary LLM () to build a keyword + ≤8-line digest. /// public sealed class GlobalWikiDigestGenerator : IGlobalWikiDigestGenerator { - public const int MaxLines = 7; + public const int MaxLines = 8; public const int MaxChars = 2_000; private readonly ILlmAdapterResolver _adapterResolver; diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs index ca81418..abc1c68 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiScoring.cs @@ -9,24 +9,30 @@ public static class GlobalWikiScoring public static IEnumerable<(GlobalWikiDocument Document, double Score)> ScoreMatches( IReadOnlyList docs, string query, - GlobalWikiAliasLexicon? lexicon = null) + GlobalWikiAliasLexicon? lexicon = null, + bool digestOnly = false) { var expansion = (lexicon ?? GlobalWikiAliasLexicon.Empty).Expand(query); return docs - .Select(d => (Document: d, Score: ScoreDocument(d, expansion))) + .Select(d => (Document: d, Score: ScoreDocument(d, expansion, digestOnly))) .Where(x => expansion.Groups.Count == 0 || x.Score > 0) .OrderByDescending(x => x.Score) - .ThenBy(x => x.Document.Content.Length); + .ThenBy(x => digestOnly ? (x.Document.Summary ?? string.Empty).Length : x.Document.Content.Length); } - public static double ScoreDocument(GlobalWikiDocument doc, GlobalWikiQueryExpansion expansion) + public static double ScoreDocument( + GlobalWikiDocument doc, + GlobalWikiQueryExpansion expansion, + bool digestOnly = false) { var score = 0.0; if (expansion.Groups.Count > 0) { var identity = $"{doc.DocumentId} {doc.Slug} {doc.Title}".ToLowerInvariant(); var summary = (doc.Summary ?? string.Empty).ToLowerInvariant(); - var body = $"{doc.Summary} {doc.Content} {doc.SourceId}".ToLowerInvariant(); + var body = digestOnly + ? $"{doc.Summary} {doc.SourceId}".ToLowerInvariant() + : $"{doc.Summary} {doc.Content} {doc.SourceId}".ToLowerInvariant(); if (expansion.OriginalPhrase.Length >= 4 && body.Contains(expansion.OriginalPhrase, StringComparison.Ordinal)) @@ -38,16 +44,21 @@ public static double ScoreDocument(GlobalWikiDocument doc, GlobalWikiQueryExpans score += 100; else if (group.Hits(summary)) { - score += summary.Contains("keywords:", StringComparison.Ordinal) + score += summary.Contains("keywords:", StringComparison.OrdinalIgnoreCase) ? 35 : 20; } - else if (group.Hits(body)) + else if (!digestOnly && group.Hits(body)) score += 10; } + score += ScoreQuestionOverlap(doc, expansion); + if (GlobalWikiCatalog.IsCatalogDocument(doc.DocumentId)) score += 15; + + if (score == 0) + return 0; } var ageHours = (DateTimeOffset.UtcNow - doc.UpdatedAt).TotalHours; @@ -55,6 +66,36 @@ public static double ScoreDocument(GlobalWikiDocument doc, GlobalWikiQueryExpans return score; } + private static double ScoreQuestionOverlap(GlobalWikiDocument doc, GlobalWikiQueryExpansion expansion) + { + if (expansion.Groups.Count == 0) + return 0; + + var queryTokens = expansion.Groups + .Select(g => g.Canonical) + .Where(t => t.Length >= 2) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + if (queryTokens.Count == 0) + return 0; + + var best = 0.0; + foreach (var phrase in GlobalWikiDigestFields.ExtractQuestionPhrases(doc.Summary)) + { + var phraseTokens = Tokenize(phrase); + if (phraseTokens.Count == 0) + continue; + + var overlap = queryTokens.Count(t => phraseTokens.Contains(t)); + if (overlap == 0) + continue; + + var ratio = (double)overlap / Math.Max(queryTokens.Count, phraseTokens.Count); + best = Math.Max(best, 30 + ratio * 40); + } + + return best; + } + public static double ScoreDocument(GlobalWikiDocument doc, HashSet tokens, string phrase) => ScoreDocument( doc, @@ -77,7 +118,7 @@ public static HashSet Tokenize(string? text) if (string.IsNullOrWhiteSpace(text)) return []; - return text.Split([' ', '\t', '\n', '\r', ',', '.', ':', ';', '/', '|', '(', ')', '[', ']'], + return text.Split([' ', '\t', '\n', '\r', ',', '.', ':', ';', '/', '|', '(', ')', '[', ']', '?'], StringSplitOptions.RemoveEmptyEntries) .Select(t => t.Trim().ToLowerInvariant()) .Where(t => t.Length >= 2) diff --git a/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs b/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs index ab6fc77..100c775 100644 --- a/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs +++ b/src/ContextMemory.Core/GlobalWiki/GlobalWikiService.cs @@ -225,7 +225,14 @@ public async Task QueryAsync( : request.DigestOnly ? DefaultDigestBudgetChars : DefaultBudgetChars; var matchedDocs = (await _store - .SearchAsync(appId, request.Query, asOf, request.SourceId, topK, cancellationToken) + .SearchAsync( + appId, + request.Query, + asOf, + request.SourceId, + topK, + request.DigestOnly, + cancellationToken) .ConfigureAwait(false)) .Where(d => !GlobalWikiCatalog.IsGlossaryDocument(d.DocumentId)) .ToList(); @@ -250,7 +257,11 @@ public async Task QueryAsync( if (matchedDocs.Count > 0) { var lexicon = GlobalWikiAliasLexicon.ForApp(appId); - var rescored = GlobalWikiScoring.ScoreMatches(matchedDocs, request.Query, lexicon).ToList(); + var rescored = GlobalWikiScoring.ScoreMatches( + matchedDocs, + request.Query, + lexicon, + request.DigestOnly).ToList(); matchedDocs = rescored.Select(x => x.Document).ToList(); matches = rescored .Select(m => new GlobalWikiMatch diff --git a/src/ContextMemory.Core/Localization/LlmPrompts.cs b/src/ContextMemory.Core/Localization/LlmPrompts.cs index eddcea7..5047dfa 100644 --- a/src/ContextMemory.Core/Localization/LlmPrompts.cs +++ b/src/ContextMemory.Core/Localization/LlmPrompts.cs @@ -234,11 +234,12 @@ public static string WikiTicketDigest(string? language) => FULL CONTENT (includes comments — extract rules/policies from them): {content} - Write at most 7 lines of plain text (no markdown fences, no JSON): + Write at most 8 lines of plain text (no markdown fences, no JSON): - Line 1 MUST be: Keywords: k1, k2, k3, ... (5–12 specific terms: IDs, systems, rules, entities) - When the document uses an acronym, include BOTH forms in Keywords, e.g. VVE (Virtual Vehicle Enablement) - Line 2 (only when acronyms appear in the document): Aliases: ACRO=Full expansion; ACRO2=Other expansion - - Lines 3–7: short factual bullets covering problem, decisions, and especially RULES found in comments/discussion + - Line 3 (optional, 1–3 intent questions this doc answers): Questions: short question one? | short question two? + - Lines 4–8: short factual bullets covering problem, decisions, and especially RULES found in comments/discussion - Prefer concrete constraints ("must", "cannot", SLAs, formulas, owners) over vague summary - Same language as the document - Do not invent facts; omit unknowns @@ -253,11 +254,12 @@ Resume este documento da base de conhecimento para pesquisa. CONTEÚDO COMPLETO (inclui comentários — extrai regras/políticas deles): {content} - Escreve no máximo 7 linhas de texto simples (sem fences markdown, sem JSON): + Escreve no máximo 8 linhas de texto simples (sem fences markdown, sem JSON): - A linha 1 DEVE ser: Keywords: k1, k2, k3, ... (5–12 termos específicos: IDs, sistemas, regras, entidades) - Quando o documento usa uma sigla, inclui as DUAS formas em Keywords, p.ex. VVE (Virtual Vehicle Enablement) - Linha 2 (só se houver siglas no documento): Aliases: SIGLA=Expansão completa; SIGLA2=Outra expansão - - Linhas 3–7: factos curtos sobre problema, decisões e sobretudo REGRAS nos comentários/discussão + - Linha 3 (opcional, 1–3 perguntas de intenção que este doc responde): Questions: pergunta curta? | outra pergunta? + - Linhas 4–8: factos curtos sobre problema, decisões e sobretudo REGRAS nos comentários/discussão - Prefere restrições concretas ("deve", "não pode", SLAs, fórmulas, owners) a resumos vagos - Mesma língua do documento - Não inventes factos; omite o que não souberes diff --git a/src/ContextMemory.Infrastructure/Migrations/20260817120000_AddGlobalWikiDigestSearchVector.cs b/src/ContextMemory.Infrastructure/Migrations/20260817120000_AddGlobalWikiDigestSearchVector.cs new file mode 100644 index 0000000..a17cb7b --- /dev/null +++ b/src/ContextMemory.Infrastructure/Migrations/20260817120000_AddGlobalWikiDigestSearchVector.cs @@ -0,0 +1,42 @@ +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ContextMemory.Infrastructure.Migrations; + +[DbContext(typeof(Persistence.Postgres.ContextMemoryDbContext))] +[Migration("20260817120000_AddGlobalWikiDigestSearchVector")] +public partial class AddGlobalWikiDigestSearchVector : Migration +{ + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + ALTER TABLE global_wiki_documents + ADD COLUMN IF NOT EXISTS digest_search_vector tsvector + GENERATED ALWAYS AS ( + to_tsvector( + 'simple', + coalesce("DocumentId",'') || ' ' || + coalesce("Slug",'') || ' ' || + coalesce("Title",'') || ' ' || + coalesce("Summary",'') || ' ' || + coalesce("SourceId",'') + ) + ) STORED; + """); + + migrationBuilder.Sql(""" + CREATE INDEX IF NOT EXISTS IX_global_wiki_documents_digest_search_vector + ON global_wiki_documents USING GIN (digest_search_vector); + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("""DROP INDEX IF EXISTS IX_global_wiki_documents_digest_search_vector;"""); + migrationBuilder.Sql("""ALTER TABLE global_wiki_documents DROP COLUMN IF EXISTS digest_search_vector;"""); + } +} diff --git a/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs b/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs index b161748..a48fe08 100644 --- a/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs +++ b/src/ContextMemory.Infrastructure/Persistence/Postgres/PostgresGlobalWikiStore.cs @@ -286,6 +286,7 @@ public async Task> SearchAsync( DateTimeOffset? asOf = null, string? sourceId = null, int topK = 50, + bool digestOnly = false, CancellationToken cancellationToken = default) { topK = Math.Clamp(topK, 1, 200); @@ -295,69 +296,91 @@ public async Task> SearchAsync( var lexicon = await LoadLexiconAsync(db, appId, point, cancellationToken).ConfigureAwait(false); GlobalWikiAliasLexicon.Remember(appId, lexicon); var expansion = lexicon.Expand(query); - var tsQuery = GlobalWikiAliasLexicon.ToPostgresTsQuery(expansion); + + if (expansion.Groups.Count == 0) + return []; + + var vectorColumn = digestOnly ? "digest_search_vector" : "search_vector"; + var strictQuery = GlobalWikiAliasLexicon.ToPostgresTsQuery(expansion); + var orQuery = GlobalWikiAliasLexicon.ToPostgresOrTsQuery(expansion); List entities; try { - if (expansion.Groups.Count == 0) - { - return (await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false)) - .Take(topK) - .ToList(); - } + entities = await QueryFtsAsync( + db, + appId, + point, + sourceId, + vectorColumn, + strictQuery, + topK, + cancellationToken) + .ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(tsQuery)) + if (entities.Count == 0 && !string.IsNullOrWhiteSpace(orQuery) + && !string.Equals(orQuery, strictQuery, StringComparison.Ordinal)) { - var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); - return GlobalWikiScoring.ScoreMatches(docs, query, lexicon) - .Take(topK) - .Select(x => x.Document) - .ToList(); + entities = await QueryFtsAsync( + db, + appId, + point, + sourceId, + vectorColumn, + orQuery, + topK, + cancellationToken) + .ConfigureAwait(false); } - - // Synonym groups as OR, AND across original query terms. Safer than raw user plainto_tsquery. - var sql = """ - SELECT * FROM global_wiki_documents - WHERE "AppId" = {0} - AND "ValidFrom" <= {1} - AND ("ValidTo" IS NULL OR "ValidTo" > {1}) - AND ({2}::text IS NULL OR "SourceId" = {2}) - AND search_vector @@ to_tsquery('simple', {3}) - ORDER BY ts_rank(search_vector, to_tsquery('simple', {3})) DESC - LIMIT {4} - """; - - entities = await db.GlobalWikiDocuments - .FromSqlRaw(sql, appId, point, (object?)sourceId ?? DBNull.Value, tsQuery, topK) - .AsNoTracking() - .ToListAsync(cancellationToken) - .ConfigureAwait(false); } catch { - var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); - return GlobalWikiScoring.ScoreMatches(docs, query, lexicon) - .Take(topK) - .Select(x => x.Document) - .ToList(); + return []; } if (entities.Count == 0) - { - var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); - return GlobalWikiScoring.ScoreMatches(docs, query, lexicon) - .Take(topK) - .Select(x => x.Document) - .ToList(); - } + return []; - return GlobalWikiScoring.ScoreMatches(entities.Select(ToDocument).ToList(), query, lexicon) + return GlobalWikiScoring.ScoreMatches( + entities.Select(ToDocument).ToList(), + query, + lexicon, + digestOnly) .Take(topK) .Select(x => x.Document) .ToList(); } + private static async Task> QueryFtsAsync( + ContextMemoryDbContext db, + string appId, + DateTimeOffset point, + string? sourceId, + string vectorColumn, + string? tsQuery, + int topK, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(tsQuery)) + return []; + + var sql = + "SELECT * FROM global_wiki_documents " + + "WHERE \"AppId\" = {0} " + + "AND \"ValidFrom\" <= {1} " + + "AND (\"ValidTo\" IS NULL OR \"ValidTo\" > {1}) " + + "AND ({2}::text IS NULL OR \"SourceId\" = {2}) " + + $"AND {vectorColumn} @@ to_tsquery('simple', {{3}}) " + + $"ORDER BY ts_rank({vectorColumn}, to_tsquery('simple', {{3}})) DESC " + + "LIMIT {4}"; + + return await db.GlobalWikiDocuments + .FromSqlRaw(sql, appId, point, (object?)sourceId ?? DBNull.Value, tsQuery, topK) + .AsNoTracking() + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + } + private static async Task LoadLexiconAsync( ContextMemoryDbContext db, string appId, diff --git a/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs b/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs index 973319a..bca5d7a 100644 --- a/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs +++ b/src/ContextMemory.Infrastructure/Wiki/FileGlobalWikiStore.cs @@ -301,18 +301,18 @@ public async Task> SearchAsync( DateTimeOffset? asOf = null, string? sourceId = null, int topK = 50, + bool digestOnly = false, CancellationToken cancellationToken = default) { var docs = await GetAllForQueryAsync(appId, sourceId, asOf, cancellationToken).ConfigureAwait(false); - var index = await ReadIndexAsync(appId, cancellationToken).ConfigureAwait(false); + var index = await ReadIndexAsync(appId, digestOnly, cancellationToken).ConfigureAwait(false); var lexicon = GlobalWikiAliasLexicon.FromDocuments(docs); GlobalWikiAliasLexicon.Remember(appId, lexicon); var expansion = lexicon.Expand(query); var tokens = expansion.IndexTokens.ToHashSet(StringComparer.OrdinalIgnoreCase); if (tokens.Count == 0) - return docs.Take(Math.Max(1, topK)).ToList(); + return []; - // Prefer inverted index postings when available, then fall back to full scoring set. HashSet? candidateIds = null; if (index is { Count: > 0 }) { @@ -329,9 +329,12 @@ public async Task> SearchAsync( var pool = candidateIds is { Count: > 0 } ? docs.Where(d => candidateIds.Contains(d.DocumentId)).ToList() - : docs; + : []; + + if (pool.Count == 0) + return []; - return GlobalWikiScoring.ScoreMatches(pool, query, lexicon) + return GlobalWikiScoring.ScoreMatches(pool, query, lexicon, digestOnly) .Take(Math.Clamp(topK, 1, 200)) .Select(x => x.Document) .ToList(); @@ -448,42 +451,63 @@ private async Task MigrateLegacyAsync(string appId, CancellationToken cancellati private async Task RebuildIndexAsync(string appId, CancellationToken cancellationToken) { var docs = await GetAllForQueryAsync(appId, sourceId: null, asOf: null, cancellationToken).ConfigureAwait(false); - var index = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var fullIndex = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var digestIndex = new Dictionary>(StringComparer.OrdinalIgnoreCase); foreach (var doc in docs) { - var text = $"{doc.DocumentId} {doc.Slug} {doc.Title} {doc.Summary} {doc.SourceId} {doc.Content}"; - foreach (var token in Tokenize(text)) - { - if (!index.TryGetValue(token, out var set)) - { - set = new HashSet(StringComparer.OrdinalIgnoreCase); - index[token] = set; - } + IndexDocumentTokens(fullIndex, doc.DocumentId, $"{doc.DocumentId} {doc.Slug} {doc.Title} {doc.Summary} {doc.SourceId} {doc.Content}"); + IndexDocumentTokens(digestIndex, doc.DocumentId, GlobalWikiDigestFields.DigestIndexText(doc)); + } - set.Add(doc.DocumentId); + Directory.CreateDirectory(GetAppDir(appId)); + await WriteIndexFileAsync(GetIndexPath(appId), fullIndex, cancellationToken).ConfigureAwait(false); + await WriteIndexFileAsync(GetDigestIndexPath(appId), digestIndex, cancellationToken).ConfigureAwait(false); + } + + private static void IndexDocumentTokens( + Dictionary> index, + string documentId, + string text) + { + foreach (var token in GlobalWikiScoring.Tokenize(text)) + { + if (!index.TryGetValue(token, out var set)) + { + set = new HashSet(StringComparer.OrdinalIgnoreCase); + index[token] = set; } + + set.Add(documentId); } + } + private static async Task WriteIndexFileAsync( + string path, + Dictionary> index, + CancellationToken cancellationToken) + { var serializable = index.ToDictionary( kv => kv.Key, kv => kv.Value.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToList(), StringComparer.OrdinalIgnoreCase); - Directory.CreateDirectory(GetAppDir(appId)); - await File.WriteAllTextAsync( - GetIndexPath(appId), - JsonSerializer.Serialize(serializable, JsonOptions), - cancellationToken) + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(serializable, JsonOptions), cancellationToken) .ConfigureAwait(false); } private async Task>?> ReadIndexAsync( string appId, + bool digestOnly, CancellationToken cancellationToken) { - var path = GetIndexPath(appId); + var path = digestOnly ? GetDigestIndexPath(appId) : GetIndexPath(appId); if (!File.Exists(path)) + { + if (digestOnly) + return await ReadIndexAsync(appId, digestOnly: false, cancellationToken).ConfigureAwait(false); return null; + } + try { var json = await File.ReadAllTextAsync(path, cancellationToken).ConfigureAwait(false); @@ -495,7 +519,8 @@ await File.WriteAllTextAsync( } } - private static HashSet Tokenize(string? text) => GlobalWikiScoring.Tokenize(text); + private string GetDigestIndexPath(string appId) => + Path.Combine(GetAppDir(appId), "digest-index.json"); private static bool MetadataEquals(Dictionary a, Dictionary b) { diff --git a/tests/ContextMemory.Api.Tests/GlobalWikiDigestSearchTests.cs b/tests/ContextMemory.Api.Tests/GlobalWikiDigestSearchTests.cs new file mode 100644 index 0000000..f7f66ca --- /dev/null +++ b/tests/ContextMemory.Api.Tests/GlobalWikiDigestSearchTests.cs @@ -0,0 +1,70 @@ +using ContextMemory.Core.GlobalWiki; +using ContextMemory.Core.Models; +using Xunit; + +namespace ContextMemory.Api.Tests; + +public sealed class GlobalWikiDigestSearchTests +{ + [Fact] + public void ScoreMatches_QuestionOverlapBoostsDigestMatch() + { + var doc = new GlobalWikiDocument + { + AppId = "demo", + DocumentId = "fleet-eu", + Slug = "fleet-eu", + Title = "EU fleet", + Summary = """ + Keywords: fleet, EU + Questions: when to enable virtual vehicle? | fleet rollout deadline EU + VVE required before SOP. + """, + Content = "SECRET_BODY_ONLY_TOKEN", + UpdatedAt = DateTimeOffset.UtcNow + }; + + var ranked = GlobalWikiScoring.ScoreMatches([doc], "virtual vehicle enable EU", digestOnly: true) + .Select(x => x.Document.DocumentId) + .ToList(); + + Assert.Equal("fleet-eu", ranked[0]); + Assert.True(GlobalWikiScoring.ScoreDocument(doc, GlobalWikiAliasLexicon.Empty.Expand("virtual vehicle"), digestOnly: true) > 0); + } + + [Fact] + public void DigestOnly_IgnoresBodyOnlyTerms() + { + const string secret = "SECRET_BODY_ONLY_TOKEN_XYZ"; + var doc = new GlobalWikiDocument + { + AppId = "demo", + DocumentId = "secret-doc", + Slug = "secret-doc", + Title = "Title", + Summary = "Keywords: public, fleet", + Content = secret, + UpdatedAt = DateTimeOffset.UtcNow + }; + + var ranked = GlobalWikiScoring.ScoreMatches([doc], secret, digestOnly: true).ToList(); + Assert.Empty(ranked); + } + + [Fact] + public void ToPostgresOrTsQuery_JoinsTokensWithOr() + { + var expansion = GlobalWikiAliasLexicon.Empty.Expand("virtual vehicle enablement"); + var orQuery = GlobalWikiAliasLexicon.ToPostgresOrTsQuery(expansion); + Assert.NotNull(orQuery); + Assert.Contains("|", orQuery); + } + + [Fact] + public void ExtractQuestionPhrases_ParsesPipeSeparatedList() + { + var phrases = GlobalWikiDigestFields.ExtractQuestionPhrases( + "Keywords: fleet\nQuestions: when to enable VVE? | EU rollout deadline"); + Assert.Equal(2, phrases.Count); + } +} diff --git a/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs b/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs index 05d2f20..a3c9022 100644 --- a/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs +++ b/tests/ContextMemory.Api.Tests/GlobalWikiServiceTests.cs @@ -325,6 +325,39 @@ public async Task Query_ResolvesAcronymsViaGlossary() } } + [Fact] + public async Task Query_DigestOnly_DoesNotMatchBodyOnlyTerms() + { + var root = Path.Combine(Path.GetTempPath(), "cm-global-wiki-" + Guid.NewGuid().ToString("N")); + try + { + var service = CreateService(root); + const string secretBody = "SECRET_BODY_ONLY_DIGEST_ISOLATION"; + await service.UpsertAsync("demo", "body-only", new GlobalWikiUpsertRequest + { + Title = "Body only", + Content = $"# Body\n\n{secretBody}", + Summary = "Keywords: public, fleet\nQuestions: fleet policy overview?", + SourceId = "confluence:DOCS" + }); + + var result = await service.QueryAsync("demo", new GlobalWikiQueryRequest + { + Query = secretBody, + TopK = 5, + BudgetChars = 2000, + DigestOnly = true + }); + + Assert.DoesNotContain(result.Matches, m => m.DocumentId == "body-only"); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + [Fact] public async Task Query_PacksMatchedDocumentBody_BeforeFillerIndexPages() { @@ -379,7 +412,7 @@ public async Task Query_PacksMatchedDocumentBody_BeforeFillerIndexPages() } [Fact] - public void NormalizeDigest_EnforcesKeywordsLine_AndMaxSevenLines() + public void NormalizeDigest_EnforcesKeywordsLine_AndMaxEightLines() { var raw = """ @@ -390,15 +423,16 @@ Extra 2 Extra 3 Extra 4 Extra 5 - Extra 6 should drop + Extra 6 + Extra 7 should drop """; var normalized = GlobalWikiDigestGenerator.NormalizeDigest(raw, "PAC-668", "Billing"); var lines = normalized.Split('\n'); - Assert.Equal(7, lines.Length); + Assert.Equal(8, lines.Length); Assert.StartsWith("Keywords:", lines[0]); Assert.Contains("never reopen closed batches", normalized); - Assert.DoesNotContain("Extra 6 should drop", normalized); + Assert.DoesNotContain("Extra 7 should drop", normalized); } [Fact]