diff --git a/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs b/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs index ee2437d9b..ee409fa70 100644 --- a/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs +++ b/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs @@ -4,6 +4,7 @@ using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Tools.RAG; +using AIStudio.Tools.Security; using AIStudio.Tools.Services; namespace AIStudio.Agents; @@ -237,7 +238,20 @@ public async Task ValidateRetrievalContextAsyn // 2. Prepare the retrieval context for the agent: // var additionalData = new Dictionary(); - var markdownRetrievalContext = await retrievalContext.AsMarkdown(token: token); + string markdownRetrievalContext; + try + { + markdownRetrievalContext = await retrievalContext.AsMarkdown(token: token); + } + catch (PromptInjectionBlockedException exception) + { + logger.LogWarning( + exception, + "Blocked retrieval context '{DataSourceName}' at '{Path}' before validation.", + retrievalContext.DataSourceName, + retrievalContext.Path); + return new(false, "The retrieval context was blocked due to suspected prompt injection.", 1.0f, retrievalContext); + } additionalData.Add("retrievalContext", markdownRetrievalContext); // diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index 395f8055f..6e911a90d 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -5,6 +5,7 @@ using AIStudio.Tools.AIJobs; using AIStudio.Tools.AssistantSessions; using AIStudio.Tools.Media; +using AIStudio.Tools.Security; using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; @@ -273,6 +274,12 @@ private async Task Start() this.Logger.LogError(e, "The provider request failed for assistant '{AssistantTitle}'. Status={StatusCode}, Reason='{ReasonPhrase}', Body='{ResponseBody}'", this.Title, e.StatusCode, e.ReasonPhrase, e.ResponseBody); await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage)); } + catch (PromptInjectionBlockedException e) + { + sessionStatus = AssistantSessionStatus.FAILED; + errorMessage = e.Message; + this.Logger.LogWarning(e, "Blocked prompt-injection content during assistant session '{AssistantTitle}'.", this.Title); + } catch (Exception e) { sessionStatus = AssistantSessionStatus.FAILED; @@ -461,6 +468,18 @@ protected async Task AddAIResponseAsync(DateTimeOffset time, bool hideCo return string.Empty; } + catch (PromptInjectionBlockedException e) + { + this.Logger.LogWarning(e, "Blocked prompt-injection content before sending assistant request for '{AssistantTitle}'.", this.Title); + + if (this.ResultingContentBlock is not null && string.IsNullOrWhiteSpace(aiText.Text)) + { + this.ChatThread?.Blocks.Remove(this.ResultingContentBlock); + this.ResultingContentBlock = null; + } + + return string.Empty; + } finally { this.IsProcessing = this.assistantSessionId is not null && (this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey)?.IsActive ?? false); diff --git a/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs b/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs index 0fed44516..5b674b1fb 100644 --- a/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs +++ b/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs @@ -716,7 +716,7 @@ private async Task PromptLoadDocumentsContent() continue; } - var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue); + var fileContent = await UserFile.LoadFileData(document.FilePath, this.RustService, this.DialogService); sb.AppendLine($""" ## DOCUMENT {numDocuments}: diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 3c9473b0a..948ace26e 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3040,6 +3040,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2250917004"] = "Some fi -- Drag and drop files into the marked area or click here to attach documents: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T230755331"] = "Drag and drop files into the marked area or click here to attach documents:" +-- The file '{0}' could not be checked for prompt injection and was not attached. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2404630365"] = "The file '{0}' could not be checked for prompt injection and was not attached." + -- The media transcription was canceled. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T241403726"] = "The media transcription was canceled." @@ -5572,6 +5575,78 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T384594633"] = "Th -- Prompting Guideline UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T4250996615"] = "Prompting Guideline" +-- Chat attachment +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1071345316"] = "Chat attachment" + +-- Content source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1129278507"] = "Content source" + +-- Attempt to override instructions +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T161976090"] = "Attempt to override instructions" + +-- Security warning +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1862129203"] = "Security warning" + +-- Attempt to expose protected data +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2050274293"] = "Attempt to expose protected data" + +-- Attempt to bypass safeguards +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2260642992"] = "Attempt to bypass safeguards" + +-- Attempt to change the AI's role +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2340508370"] = "Attempt to change the AI's role" + +-- Hidden instructions using markup +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2526538070"] = "Hidden instructions using markup" + +-- Web content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2626468388"] = "Web content" + +-- Source type +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T280442848"] = "Source type" + +-- Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3122726298"] = "Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content." + +-- Hidden instructions using delimiters +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T329656456"] = "Hidden instructions using delimiters" + +-- Retrieved context +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3347144620"] = "Retrieved context" + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Close" + +-- Attempt to manipulate an agent +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T355252317"] = "Attempt to manipulate an agent" + +-- AI Studio has reliably detected and blocked the suspicious content. Your applications and data remain protected. Please review the content you uploaded. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3661810823"] = "AI Studio has reliably detected and blocked the suspicious content. Your applications and data remain protected. Please review the content you uploaded." + +-- File content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3788064862"] = "File content" + +-- Persistent or delayed instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4169123215"] = "Persistent or delayed instruction" + +-- Typical attacks on AI systems (e.g. prompt injection) hide instructions within untrusted content to trick an AI model into ignoring its intended rules or performing unintended actions. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4221674400"] = "Typical attacks on AI systems (e.g. prompt injection) hide instructions within untrusted content to trick an AI model into ignoring its intended rules or performing unintended actions." + +-- Detected content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4223810750"] = "Detected content" + +-- More information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "More information" + +-- Hidden instructions using encoding +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T49495195"] = "Hidden instructions using encoding" + +-- Hide more information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T808738984"] = "Hide more information" + +-- Obfuscated instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T87316699"] = "Obfuscated instruction" + -- Hugging Face Inference Provider UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1085481431"] = "Hugging Face Inference Provider" @@ -6052,6 +6127,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T6790 -- When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T711745239"] = "When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model." +-- Protect against prompt injection in external content? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1040385578"] = "Protect against prompt injection in external content?" + -- Preselect one of your chat templates? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1402022556"] = "Preselect one of your chat templates?" @@ -6073,12 +6151,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1915793195"] -- Preselect a profile UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2322771068"] = "Preselect a profile" +-- A blocking alert explains the detected attack pattern +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2341755932"] = "A blocking alert explains the detected attack pattern" + -- Apply default data source option when sending assistant results to chat UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2510376349"] = "Apply default data source option when sending assistant results to chat" -- Control how the LLM provider for added chats is selected. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T263621180"] = "Control how the LLM provider for added chats is selected." +-- Show a learning alert when prompt injection is detected? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2754886869"] = "Show a learning alert when prompt injection is detected?" + -- Provider selection when loading a chat and sending assistant results to chat UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2868379953"] = "Provider selection when loading a chat and sending assistant results to chat" @@ -6088,6 +6172,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2913693228"] -- Do you want to use any shortcut to send your input? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2936560092"] = "Do you want to use any shortcut to send your input?" +-- Shows an explanation dialog with an external reference when AI Studio blocks suspicious content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3129787554"] = "Shows an explanation dialog with an external reference when AI Studio blocks suspicious content." + -- Would you like to set one of your chat templates as the default for chats? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3234927721"] = "Would you like to set one of your chat templates as the default for chats?" @@ -6097,6 +6184,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3383186996"] -- Close UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3448155331"] = "Close" +-- Only the block notification is shown +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3491678837"] = "Only the block notification is shown" + -- First (oldest) message is shown, after loading a chat UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3507181366"] = "First (oldest) message is shown, after loading a chat" @@ -6109,9 +6199,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3730599555"] -- Latest message is shown, after loading a chat UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3755993611"] = "Latest message is shown, after loading a chat" +-- Potential prompt injections are blocked before they reach an LLM +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3982766677"] = "Potential prompt injections are blocked before they reach an LLM" + -- Do you want to apply the default data source options when sending assistant results to chat? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4033153439"] = "Do you want to apply the default data source options when sending assistant results to chat?" +-- External content is passed through without prompt-injection checks +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4245463281"] = "External content is passed through without prompt-injection checks" + -- When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T477675197"] = "When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider." @@ -6121,6 +6217,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T492357592"] = -- When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T582516016"] = "When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown." +-- Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T69192586"] = "Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent." + -- Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHATTEMPLATE::T1172171653"] = "Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them." @@ -7246,6 +7345,9 @@ UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2979224202"] = "Writer" -- Show details UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3692372066"] = "Show details" +-- Security notice +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T4004397997"] = "Security notice" + -- Information UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T4256323669"] = "Information" @@ -9487,6 +9589,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document" -- Plugin archive UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive" +-- AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONGUARDSERVICE::T3560909296"] = "AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt." + -- The Assistant Builder context could not be loaded. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T108292972"] = "The Assistant Builder context could not be loaded." diff --git a/app/MindWork AI Studio/Assistants/PromptOptimizer/AssistantPromptOptimizer.razor.cs b/app/MindWork AI Studio/Assistants/PromptOptimizer/AssistantPromptOptimizer.razor.cs index ac56f2c55..2a200e4b0 100644 --- a/app/MindWork AI Studio/Assistants/PromptOptimizer/AssistantPromptOptimizer.razor.cs +++ b/app/MindWork AI Studio/Assistants/PromptOptimizer/AssistantPromptOptimizer.razor.cs @@ -4,6 +4,7 @@ using AIStudio.Chat; using AIStudio.Dialogs; using AIStudio.Dialogs.Settings; +using AIStudio.Tools.Security; using AIStudio.Tools.AssistantSessions; using Microsoft.AspNetCore.Components; @@ -583,6 +584,10 @@ private async Task LoadCustomPromptGuidelineContentAsync(FileAttachment fileAtta if (string.IsNullOrWhiteSpace(this.customPromptingGuidelineContent)) await this.MessageBus.SendWarning(new(Icons.Material.Filled.Description, T("The custom prompt guide file is empty or could not be read."))); } + catch (PromptInjectionBlockedException) + { + this.customPromptingGuidelineContent = string.Empty; + } catch { this.customPromptingGuidelineContent = string.Empty; diff --git a/app/MindWork AI Studio/Assistants/SlideBuilder/SlideAssistant.razor.cs b/app/MindWork AI Studio/Assistants/SlideBuilder/SlideAssistant.razor.cs index 1782b26db..78abad6f3 100644 --- a/app/MindWork AI Studio/Assistants/SlideBuilder/SlideAssistant.razor.cs +++ b/app/MindWork AI Studio/Assistants/SlideBuilder/SlideAssistant.razor.cs @@ -2,11 +2,15 @@ using AIStudio.Chat; using AIStudio.Dialogs.Settings; using AIStudio.Tools.AssistantSessions; +using Microsoft.AspNetCore.Components; namespace AIStudio.Assistants.SlideBuilder; public partial class SlideAssistant : AssistantBaseCore { + [Inject] + private IDialogService DialogService { get; init; } = null!; + protected override Tools.Components Component => Tools.Components.SLIDE_BUILDER_ASSISTANT; protected override string Title => T("Slide Planner Assistant"); @@ -382,7 +386,7 @@ private async Task PromptLoadDocumentsContent() continue; } - var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue); + var fileContent = await UserFile.LoadFileData(document.FilePath, this.RustService, this.DialogService); sb.AppendLine($""" ## DOCUMENT {numDocuments}: diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index 4c8be6466..db269485f 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -5,6 +5,7 @@ using AIStudio.Settings; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.RAG.RAGProcesses; +using AIStudio.Tools.Security; namespace AIStudio.Chat; @@ -293,13 +294,17 @@ public async Task PrepareTextContentForAI() LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath); continue; } + var guardService = Program.SERVICE_PROVIDER.GetRequiredService(); + var safeFileContent = await guardService.EnsureSafeForLlmAsync( + await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue), + PromptInjectionSource.ChatAttachment(document.FilePath)); sb.AppendLine(); sb.AppendLine("---------------------------------------"); sb.AppendLine($"File path: {document.FilePath}"); sb.AppendLine("File content:"); sb.AppendLine("````"); - sb.AppendLine(await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue)); + sb.AppendLine(safeFileContent); sb.AppendLine("````"); } diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs index 4b4274fd9..6e4a151f3 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs @@ -5,6 +5,7 @@ using AIStudio.Tools.Rust; using AIStudio.Tools.Services; using AIStudio.Tools.Validation; +using AIStudio.Tools.Security; using Microsoft.AspNetCore.Components; @@ -86,7 +87,10 @@ public partial class AttachDocuments : MSGComponentBase /// Creates and persists a draft owner after media import confirmation. [Parameter] public Func> EnsureOwnerChatAsync { get; set; } = _ => Task.FromResult(null); - + + [Inject] + private PromptInjectionGuardService PromptInjectionGuardService { get; init; } = null!; + [Inject] private ILogger Logger { get; set; } = null!; @@ -457,10 +461,7 @@ await this.MessageBus.SendWarning(new( if (!canAddRegularFiles) break; - if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.ValidateMediaFileTypes, this.Provider)) - continue; - - this.DocumentPaths.Add(FileAttachment.FromPath(path)); + await this.TryAddFileAsync(path); } if (mediaPaths.Count is 0) @@ -529,4 +530,34 @@ private async Task InvestigateFile(FileAttachment fileAttachment) await this.DialogService.ShowAsync(T("Document Preview"), dialogParameters, DialogOptions.FULLSCREEN); } -} \ No newline at end of file + + private async Task TryAddFileAsync(string filePath) + { + if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, filePath, this.ValidateMediaFileTypes, this.Provider)) + return false; + + var attachment = FileAttachment.FromPath(filePath); + if (attachment.Type is FileAttachmentType.DOCUMENT && this.PromptInjectionGuardService.IsProtectionEnabled) + { + try + { + var fileContent = await this.RustService.ReadArbitraryFileData(filePath, int.MaxValue); + await this.PromptInjectionGuardService.EnsureSafeForLlmAsync(fileContent, PromptInjectionSource.ChatAttachment(filePath)); + } + catch (PromptInjectionBlockedException) + { + return false; + } + catch (Exception exception) + { + this.Logger.LogError(exception, "File attachment '{FilePath}' could not be checked for prompt injection and will not be attached.", filePath); + await this.MessageBus.SendError(new( + Icons.Material.Filled.Cancel, + string.Format(T("The file '{0}' could not be checked for prompt injection and was not attached."), attachment.FileName))); + return false; + } + } + + return this.DocumentPaths.Add(attachment); + } +} diff --git a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs index cf23d97c5..ca65f0bd9 100644 --- a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs +++ b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs @@ -3,6 +3,7 @@ using AIStudio.Tools.Rust; using AIStudio.Tools.Services; using AIStudio.Tools.Validation; +using AIStudio.Tools.Security; using Microsoft.AspNetCore.Components; @@ -329,6 +330,11 @@ private async Task LoadFileIfValid(string filePath) this.Logger.LogInformation("Successfully loaded file content: {FilePath}", filePath); return true; } + catch (PromptInjectionBlockedException) + { + this.Logger.LogWarning("Blocked suspected prompt injection while loading file content: {FilePath}", filePath); + return false; + } catch (Exception ex) { this.Logger.LogError(ex, "Failed to load file content: {FilePath}", filePath); diff --git a/app/MindWork AI Studio/Components/ReadWebContent.razor.cs b/app/MindWork AI Studio/Components/ReadWebContent.razor.cs index 53a5e616c..263fb8007 100644 --- a/app/MindWork AI Studio/Components/ReadWebContent.razor.cs +++ b/app/MindWork AI Studio/Components/ReadWebContent.razor.cs @@ -1,5 +1,6 @@ using AIStudio.Agents; using AIStudio.Chat; +using AIStudio.Tools.Security; using Microsoft.AspNetCore.Components; @@ -7,6 +8,9 @@ namespace AIStudio.Components; public partial class ReadWebContent : MSGComponentBase { + [Inject] + private PromptInjectionGuardService PromptInjectionGuardService { get; init; } = null!; + [Inject] private HTMLParser HTMLParser { get; init; } = null!; @@ -87,6 +91,7 @@ private async Task LoadFromWeb() this.processStep = this.process[ReadWebContentSteps.PARSING]; this.StateHasChanged(); markdown = this.HTMLParser.ParseToMarkdown(html); + markdown = await this.PromptInjectionGuardService.EnsureSafeForLlmAsync(markdown, PromptInjectionSource.WebContent(this.providedURL)); if (this.PreselectContentCleanerAgent && this.providerSettings != AIStudio.Settings.Provider.NONE) { @@ -120,6 +125,10 @@ private async Task LoadFromWeb() this.StateHasChanged(); } } + catch (PromptInjectionBlockedException) + { + markdown = string.Empty; + } catch { if (this.AgentIsRunning) diff --git a/app/MindWork AI Studio/Dialogs/DocumentCheckDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DocumentCheckDialog.razor.cs index 4bf306f13..46d82126a 100644 --- a/app/MindWork AI Studio/Dialogs/DocumentCheckDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DocumentCheckDialog.razor.cs @@ -2,6 +2,7 @@ using AIStudio.Components; using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; +using AIStudio.Tools.Security; namespace AIStudio.Dialogs; @@ -42,6 +43,11 @@ protected override async Task OnAfterRenderAsync(bool firstRender) this.FileContent = fileContent; } } + catch (PromptInjectionBlockedException exception) + { + this.Logger.LogWarning(exception, "Blocked suspected prompt injection while previewing '{FilePath}'", this.Document?.FilePath); + this.FileContent = string.Empty; + } catch (Exception ex) { this.Logger.LogError(ex, "Failed to load file content from '{FilePath}'", this.Document); diff --git a/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor new file mode 100644 index 000000000..6bc13dff9 --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor @@ -0,0 +1,133 @@ +@using AIStudio.Tools.Security +@inherits MSGComponentBase + + + + + + @if (Result is not null) + { + + + + + + + + + @T("Security warning") + + + + @T("AI Studio has reliably detected and blocked the suspicious content. Your applications and data remain protected. Please review the content you uploaded.") + + + + + + + + @T("Typical attacks on AI systems (e.g. prompt injection) hide instructions within untrusted content to trick an AI model into ignoring its intended rules or performing unintended actions.") + + + + + + + + + + @(showPromptInjectionInformation + ? T("Hide more information") + : T("More information")) + + + + + + + + + + + + @T("Source type") + + + + @this.GetSourceKindLabel(Result.Source.Kind) + + + + + + + + + + @T("Content source") + + + + @Result.Source.Label + + + + + + + + + + @T("Detected content") + + + @foreach (var finding in Result.Findings) + { + + + + @($"{this.GetFindingCategoryLabel(finding.Category)}") + + + + + @finding.Snippet + + + } + + + + + + + + + @T("Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content.") + + + + @PromptInjectionGuardService.WIKI_URL + + + + } + + + + + @T("Close") + + + \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor.cs b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor.cs new file mode 100644 index 000000000..c4168c893 --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor.cs @@ -0,0 +1,45 @@ +using AIStudio.Components; +using AIStudio.Tools.Security; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Dialogs; + +public partial class PromptInjectionAlertDialog : MSGComponentBase +{ + private bool showPromptInjectionInformation; + + [CascadingParameter] + private IMudDialogInstance MudDialog { get; set; } = null!; + + [Parameter] + public PromptInjectionScanResult Result { get; set; } = null!; + + private void Close() => this.MudDialog.Close(); + + private void TogglePromptInjectionInformation() => this.showPromptInjectionInformation = !this.showPromptInjectionInformation; + + private string GetSourceKindLabel(string sourceKind) => sourceKind switch + { + "Web content" => T("Web content"), + "File content" => T("File content"), + "Chat attachment" => T("Chat attachment"), + "Retrieval context" => T("Retrieved context"), + _ => sourceKind, + }; + + private string GetFindingCategoryLabel(string category) => category switch + { + "override" => T("Attempt to override instructions"), + "role_override" => T("Attempt to change the AI's role"), + "exfiltration" => T("Attempt to expose protected data"), + "jailbreak" => T("Attempt to bypass safeguards"), + "agent_manipulation" => T("Attempt to manipulate an agent"), + "delimiter_evasion" => T("Hidden instructions using delimiters"), + "markup_evasion" => T("Hidden instructions using markup"), + "encoding_evasion" => T("Hidden instructions using encoding"), + "persistence" => T("Persistent or delayed instruction"), + "evasion" => T("Obfuscated instruction"), + _ => category, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor index f80fa8576..64ccb5767 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor @@ -15,6 +15,11 @@ + + + + + diff --git a/app/MindWork AI Studio/Layout/MainLayout.razor.cs b/app/MindWork AI Studio/Layout/MainLayout.razor.cs index a28f6a5c6..7faa623a2 100644 --- a/app/MindWork AI Studio/Layout/MainLayout.razor.cs +++ b/app/MindWork AI Studio/Layout/MainLayout.razor.cs @@ -7,6 +7,7 @@ using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; +using AIStudio.Tools.Security; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Routing; @@ -71,6 +72,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan private bool startupCompleted; private bool settingsWriteProtectionWarningShown; private readonly SemaphoreSlim mandatoryInfoDialogSemaphore = new(1, 1); + private readonly SemaphoreSlim promptInjectionDialogSemaphore = new(1, 1); private IReadOnlyCollection navItems = []; @@ -112,7 +114,7 @@ protected override async Task OnInitializedAsync() this.MessageBus.ApplyFilters(this, [], [ Event.UPDATE_AVAILABLE, Event.CONFIGURATION_CHANGED, Event.COLOR_THEME_CHANGED, Event.SHOW_ERROR, - Event.SHOW_WARNING, Event.SHOW_SUCCESS, Event.SHOW_INFO, Event.STARTUP_PLUGIN_SYSTEM, Event.PLUGINS_RELOADED, + Event.SHOW_WARNING, Event.SHOW_SUCCESS, Event.SHOW_PROMPT_INJECTION_ALERT, Event.STARTUP_PLUGIN_SYSTEM, Event.PLUGINS_RELOADED, Event.INSTALL_UPDATE, Event.STARTUP_COMPLETED, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.ASSISTANT_SESSION_CHANGED, Event.ASSISTANT_SESSION_FINISHED, ]); @@ -253,6 +255,12 @@ await this.InvokeAsync(async () => success.Show(this.Snackbar); break; + + case Event.SHOW_PROMPT_INJECTION_ALERT: + if (data is PromptInjectionAlertMessage promptInjectionAlert) + await this.ShowPromptInjectionAlertAsync(promptInjectionAlert); + + break; case Event.SHOW_ERROR: if (data is DataErrorMessage error) @@ -347,6 +355,29 @@ await this.InvokeAsync(async () => } }); } + + private async Task ShowPromptInjectionAlertAsync(PromptInjectionAlertMessage alert) + { + await this.promptInjectionDialogSemaphore.WaitAsync(); + try + { + var dialogParameters = new DialogParameters + { + { x => x.Result, alert.Result }, + }; + + var dialogReference = await this.DialogService.ShowAsync( + T("Security notice"), + dialogParameters, + DialogOptions.BLOCKING_FULLSCREEN); + + await dialogReference.Result; + } + finally + { + this.promptInjectionDialogSemaphore.Release(); + } + } public Task ProcessMessageWithResult(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data) { diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index f0b86a741..c6d74dabc 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -259,6 +259,14 @@ CONFIG["SETTINGS"] = {} -- Examples are PRE_WRITER_MODE_2024 and PRE_RAG_2024. -- CONFIG["SETTINGS"]["DataApp.EnabledPreviewFeatures"] = { "PRE_RAG_2024" } +-- Configure prompt-injection protection for external content such as webpages, +-- retrieved context, and file attachments before it is sent to an LLM. +-- CONFIG["SETTINGS"]["DataChat.EnablePromptInjectionProtection"] = true + +-- Configure whether AI Studio shows a blocking explanation dialog when +-- suspicious content is detected and blocked. +-- CONFIG["SETTINGS"]["DataChat.ShowPromptInjectionAlert"] = true + -- Configure the preselected provider. -- It must be one of the provider IDs defined in CONFIG["LLM_PROVIDERS"]. -- Please note: using an empty string ("") will lock the preselected provider selection, even though no valid preselected provider is found. diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index bae6a4894..8b38751ec 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -4275,6 +4275,119 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T974954792"] = "Das Ergebn -- Are you sure you want to delete the chat '{0}' in the workspace '{1}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::WORKSPACES::T1016188706"] = "Möchten Sie den Chat „{0}“ im Arbeitsbereich „{1}“ wirklich löschen?" +-- Chat attachment +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1071345316"] = "Dateianhang" + +-- The file '{0}' could not be checked for prompt injection and was not attached. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2404630365"] = "Die Datei '{0}' konnte nicht auf Prompt-Injection überprüft werden und wurde nicht angehängt." + +-- Security warning +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1862129203"] = "Sicherheitswarnung" + + +-- AI Studio has reliably detected and blocked the suspicious content. Your applications and data remain protected. Please review the content you uploaded. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3661810823"] = "AI Studio hat den verdächtigen Inhalt zuverlässig erkannt und blockiert. Ihre Anwendungen und Daten bleiben geschützt. Bitte überprüfen Sie den von Ihnen hochgeladenen Inhalt." + +-- Security active +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4212328947"] = "Sicherheit aktiv" + +-- Typical attacks on AI systems +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4221674400"] = "Typische Angriffe auf KI-Systeme (z.B. Prompt Injection) verbergen Anweisungen in nicht vertrauenswürdigen Inhalten, um ein KI-Modell dazu zu bringen, seine vorgesehenen Regeln zu ignorieren oder ungewollte Aktionen auszuführen." + +-- Security notice +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T4004397997"] = "Sicherheitshinweis" + +-- Content source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1129278507"] = "Datei" + +-- Attempt to override instructions +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T161976090"] = "Versuch, Anweisungen zu überschreiben" + +-- Attempt to expose protected data +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2050274293"] = "Versuch, geschützte Daten offenzulegen" + +-- Attempt to bypass safeguards +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2260642992"] = "Versuch, Sicherheitsvorkehrungen zu umgehen" + +-- Attempt to change the AI's role +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2340508370"] = "Versuch, die Rolle der KI zu ändern" + +-- AI Studio blocked this content before it reached a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2378027194"] = "AI Studio hat diesen Inhalt blockiert, bevor er ein Modell oder einen Agenten erreicht hat." + +-- Hidden instructions using markup +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2526538070"] = "Versteckte Anweisungen mit Markup" + +-- Web content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2626468388"] = "Web-Inhalt" + +-- Source type +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T280442848"] = "Quelle" + +-- Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3122726298"] = "Prompt Injection ist eine Methode, um KI-Systeme wie Chatbots zu manipulieren. Dabei platziert ein Angreifer irreführende Anweisungen in Inhalten, sodass die KI diese als legitim betrachtet. Dadurch kann die KI Sicherheitsvorkehrungen ignorieren, private Informationen preisgeben oder schädliche Inhalte erzeugen." + +-- Hidden instructions using delimiters +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T329656456"] = "Versteckte Anweisungen mit Trennzeichen" + +-- Retrieved context +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3347144620"] = "Es scheint, dass der Kontext bereits bereitgestellt wurde. Bitte geben Sie den Text an, den Sie aus dem Englischen (US) ins Deutsche (Deutschland) übersetzen möchten. Ich werde sicherstellen, dass die Übersetzung präzise, natürlich klingt und für die Zielgruppe leicht verständlich ist." + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Schließen" + +-- Attempt to manipulate an agent +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T355252317"] = "Versuch, einen Agenten zu manipulieren" + +-- File content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3788064862"] = "Inhalt der Datei" + +-- Persistent or delayed instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4169123215"] = "Dauerhafte oder verzögerte Anweisung" + +-- Detected content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4223810750"] = "Erkannter Inhalt" + +-- More information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "Weitere Informationen" + +-- Hidden instructions using encoding +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T49495195"] = "Versteckte Anweisungen mit Kodierung" + +-- Hide more information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T808738984"] = "Weniger Informationen anzeigen" + +-- Obfuscated instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T87316699"] = "Verschleierte Anweisung" + +-- Protect against prompt injection in external content? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1040385578"] = "Soll der Zugriff auf externe Inhalte vor Prompt-Injection geschützt werden?" + +-- A blocking alert explains the detected attack pattern +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2341755932"] = "Eine blockierende Warnung erklärt das erkannte Angriffsmuster." + +-- Show a learning alert when prompt injection is detected? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2754886869"] = "Lernwarnung anzeigen, wenn Prompt-Injection erkannt wird?" + +-- Shows an explanation dialog with an external reference when AI Studio blocks suspicious content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3129787554"] = "Zeigt einen Erklärungsdialog mit einem externen Verweis an, wenn AI Studio verdächtige Inhalte blockiert." + +-- Only the block notification is shown +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3491678837"] = "Nur die Blockbenachrichtigung wird angezeigt" + +-- Potential prompt injections are blocked before they reach an LLM +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3982766677"] = "Potenzielle Prompt-Injections werden blockiert, bevor sie ein LLM erreichen." + +-- External content is passed through without prompt-injection checks +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4245463281"] = "Externer Inhalt wird ohne Prompt-Injection-Prüfungen durchgereicht" + +-- Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T69192586"] = "Prüft Webinhalte, Dateianhänge, Abrufkontext und ähnliche externe Eingaben auf Prompt-Injection-Muster, bevor sie an ein Modell oder einen Agenten gesendet werden." + + +-- AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONGUARDSERVICE::T3560909296"] = "AI Studio hat Inhalte von '{0}' blockiert, da es sich um einen möglichen Prompt-Injection-Versuch handelt." + -- Move chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::WORKSPACES::T1133040906"] = "Chat verschieben" diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 1809e2a8c..1b9105c47 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -4176,6 +4176,73 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T1061000046"] = "We hope this vis -- Integration of enterprise data UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T1127694951"] = "Integration of enterprise data" +-- Danger detected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1081126836"] = "Danger detected" + +-- More Information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1266371726"] = "More Information" + +-- Prompt injection is an attempt to hide instructions in untrusted content so that an AI model ignores its intended rules or performs unintended actions. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T163682410"] = "Prompt injection is an attempt to hide instructions in untrusted content so that an AI model ignores its intended rules or performs unintended actions." + +-- Source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1642243064"] = "Source" + +-- Hide More Information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1648537896"] = "Hide More Information" + +-- AI Studio blocked this content before it reached a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2378027194"] = "AI Studio blocked this content before it reached a model or agent." + +-- Prompt Injection detected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3052247348"] = "Prompt Injection detected" + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Close" + +-- Prompt injection is a trick used to manipulate AI systems like chatbots. Normally, these systems follow the rules set by their developers—such as being helpful and safe. But with prompt injection, an attacker crafts a clever input that makes the AI think it’s receiving a new, legitimate instruction. This can cause the AI to ignore its usual safeguards and do something unintended, like sharing private information or generating harmful content. The issue arises because the AI can’t always tell the difference between a trusted command and a deceptive one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3548442822"] = "Prompt injection is a trick used to manipulate AI systems like chatbots. Normally, these systems follow the rules set by their developers—such as being helpful and safe. But with prompt injection, an attacker crafts a clever input that makes the AI think it’s receiving a new, legitimate instruction. This can cause the AI to ignore its usual safeguards and do something unintended, like sharing private information or generating harmful content. The issue arises because the AI can’t always tell the difference between a trusted command and a deceptive one." + +-- Source kind +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T47437466"] = "Source kind" + +-- More information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "More information" + +-- Detected signals +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T556618402"] = "Detected signals" + + +-- Protect against prompt injection in external content? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1040385578"] = "Protect against prompt injection in external content?" + +-- A blocking alert explains the detected attack pattern +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2341755932"] = "A blocking alert explains the detected attack pattern" + +-- Show a learning alert when prompt injection is detected? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2754886869"] = "Show a learning alert when prompt injection is detected?" + +-- Shows an explanation dialog with an external reference when AI Studio blocks suspicious content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3129787554"] = "Shows an explanation dialog with an external reference when AI Studio blocks suspicious content." + +-- Only the block notification is shown +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3491678837"] = "Only the block notification is shown" + +-- Potential prompt injections are blocked before they reach an LLM +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3982766677"] = "Potential prompt injections are blocked before they reach an LLM" + +-- External content is passed through without prompt-injection checks +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4245463281"] = "External content is passed through without prompt-injection checks" + +-- Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T69192586"] = "Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent." + +-- Prompt Injection Detected +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3580322580"] = "Prompt Injection Detected" + +-- AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONGUARDSERVICE::T3560909296"] = "AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt." + -- Meet your needs UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T127032776"] = "Meet your needs" diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index 5a47a2612..d4a129f89 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -10,6 +10,7 @@ using AIStudio.Tools.PluginSystem.Assistants; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; +using AIStudio.Tools.Security; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Server.Kestrel.Core; @@ -193,6 +194,8 @@ public static async Task Main() builder.Services.AddHostedService(); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); // ReSharper disable AccessToDisposedClosure builder.Services.AddHostedService(_ => rust); @@ -300,4 +303,4 @@ public static async Task Main() PluginFactory.Dispose(); programLogger.LogInformation("The AI Studio server was stopped."); } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataChat.cs b/app/MindWork AI Studio/Settings/DataModel/DataChat.cs index 67b3b3134..7843c5e01 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataChat.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataChat.cs @@ -92,7 +92,16 @@ public DataSourceOptions PreselectedDataSourceOptions this.PreselectedDataSourceIds = [..value.PreselectedDataSourceIds]; } } + /// + /// Whether prompt-injection protection is enabled for external and attached content. + /// + public bool EnablePromptInjectionProtection { get; set; } = ManagedConfiguration.Register(configSelection, n => n.EnablePromptInjectionProtection, true); + /// + /// Whether an alert dialog should be shown when prompt-injection content is blocked. + /// + public bool ShowPromptInjectionAlert { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShowPromptInjectionAlert, true); + /// /// Should we show the latest message after loading? When false, we show the first (aka oldest) message. /// diff --git a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs index 7619b6f74..af6b45a0c 100644 --- a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs +++ b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs @@ -5,6 +5,7 @@ using AIStudio.Settings; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.RAG.RAGProcesses; +using AIStudio.Tools.Security; namespace AIStudio.Tools.AIJobs; @@ -266,6 +267,12 @@ private async Task RunChatGenerationAsync(AIJobState state) await this.CompleteChatGenerationAsync(state, AIJobStatus.FAILED, e.UserMessage); await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage)); } + catch (PromptInjectionBlockedException e) + { + logger.LogWarning(e, "Blocked prompt-injection content during chat generation job '{JobId}'.", state.Snapshot.JobId); + RemoveEmptyAIResponse(state); + await this.CompleteChatGenerationAsync(state, AIJobStatus.FAILED, e.Message); + } catch (Exception e) { logger.LogError(e, "The chat generation job '{JobId}' failed.", state.Snapshot.JobId); diff --git a/app/MindWork AI Studio/Tools/Event.cs b/app/MindWork AI Studio/Tools/Event.cs index 963540871..4e3be4580 100644 --- a/app/MindWork AI Studio/Tools/Event.cs +++ b/app/MindWork AI Studio/Tools/Event.cs @@ -72,6 +72,11 @@ public enum Event /// Requests display of a success notification. /// SHOW_SUCCESS, + + /// + /// Requests display of a prompt-injection alert dialog. + /// + SHOW_PROMPT_INJECTION_ALERT, /// /// Requests display of an informational notification. diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index 87acb0ea3..295a7b7ff 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -281,6 +281,8 @@ private bool TryProcessConfiguration(bool dryRun, out string message) ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourcesAutomaticValidation, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourceIds, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.SendToChatDataSourceBehavior, this.Id, settingsTable, dryRun); + ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.EnablePromptInjectionProtection, this.Id, settingsTable, dryRun); + ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.ShowPromptInjectionAlert, this.Id, settingsTable, dryRun); // Config: transcription provider? ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.UseTranscriptionProvider, Guid.Empty, this.Id, settingsTable, dryRun); diff --git a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs index 24b1d24eb..ca8d14c23 100644 --- a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs +++ b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs @@ -1,5 +1,5 @@ using System.Text; - +using AIStudio.Tools.Security; using AIStudio.Chat; namespace AIStudio.Tools.RAG; @@ -16,7 +16,18 @@ public static async Task AsMarkdown(this IReadOnlyList AsMarkdown(this IReadOnlyList AsMarkdown(this IRetrievalContext retrievalContext, StringBuilder? sb = null, int index = -1, int numTotalRetrievalContexts = -1, CancellationToken token = default) { sb ??= new StringBuilder(); + var contextBuilder = new StringBuilder(); switch (index) { case > 0 when numTotalRetrievalContexts is -1: - sb.AppendLine($"# Retrieval context {index}"); + contextBuilder.AppendLine($"# Retrieval context {index}"); break; case > 0 when numTotalRetrievalContexts > 0: - sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}"); + contextBuilder.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}"); break; default: - sb.AppendLine("# Retrieval context"); + contextBuilder.AppendLine("# Retrieval context"); break; } - sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); - sb.AppendLine($"Content category: {retrievalContext.Category}"); - sb.AppendLine($"Content type: {retrievalContext.Type}"); - sb.AppendLine($"Content path: {retrievalContext.Path}"); + contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); + contextBuilder.AppendLine($"Content category: {retrievalContext.Category}"); + contextBuilder.AppendLine($"Content type: {retrievalContext.Type}"); + contextBuilder.AppendLine($"Content path: {retrievalContext.Path}"); if(retrievalContext.Links.Count > 0) { - sb.AppendLine("Additional links:"); + contextBuilder.AppendLine("Additional links:"); foreach(var link in retrievalContext.Links) - sb.AppendLine($"- {link}"); + contextBuilder.AppendLine($"- {link}"); } + + var guardService = Program.SERVICE_PROVIDER.GetRequiredService(); + var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path); switch(retrievalContext) { case RetrievalTextContext textContext: - sb.AppendLine(); - sb.AppendLine("Matched text content:"); - sb.AppendLine("````"); - sb.AppendLine(textContext.MatchedText); - sb.AppendLine("````"); + contextBuilder.AppendLine(); + contextBuilder.AppendLine("Matched text content:"); + contextBuilder.AppendLine("````"); + contextBuilder.AppendLine(textContext.MatchedText); + contextBuilder.AppendLine("````"); if(textContext.SurroundingContent.Count > 0) { - sb.AppendLine(); - sb.AppendLine("Surrounding text content:"); + contextBuilder.AppendLine(); + contextBuilder.AppendLine("Surrounding text content:"); foreach(var surrounding in textContext.SurroundingContent) { - sb.AppendLine(); - sb.AppendLine("````"); - sb.AppendLine(surrounding); - sb.AppendLine("````"); + contextBuilder.AppendLine(); + contextBuilder.AppendLine("````"); + contextBuilder.AppendLine(surrounding); + contextBuilder.AppendLine("````"); } } - + await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source); break; case RetrievalImageContext imageContext: - sb.AppendLine(); - sb.AppendLine("Matched image content as base64-encoded data:"); - sb.AppendLine("````"); - sb.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image) + await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source); + contextBuilder.AppendLine(); + contextBuilder.AppendLine("Matched image content as base64-encoded data:"); + contextBuilder.AppendLine("````"); + contextBuilder.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image) ? base64Image : string.Empty); - sb.AppendLine("````"); + contextBuilder.AppendLine("````"); break; default: + await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source); LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet."); break; } - sb.AppendLine(); + contextBuilder.AppendLine(); + sb.Append(contextBuilder); return sb.ToString(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionAlertMessage.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionAlertMessage.cs new file mode 100644 index 000000000..170080253 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionAlertMessage.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Security; + +public sealed record PromptInjectionAlertMessage(PromptInjectionScanResult Result); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionBlockedException.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionBlockedException.cs new file mode 100644 index 000000000..ef19096cd --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionBlockedException.cs @@ -0,0 +1,6 @@ +namespace AIStudio.Tools.Security; + +public sealed class PromptInjectionBlockedException(PromptInjectionScanResult result, string message) : Exception(message) +{ + public PromptInjectionScanResult Result { get; } = result; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionFinding.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionFinding.cs new file mode 100644 index 000000000..864820c4d --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionFinding.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Security; + +public sealed record PromptInjectionFinding(string RuleId, string Category, string Snippet); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionGuardService.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionGuardService.cs new file mode 100644 index 000000000..017578f9e --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionGuardService.cs @@ -0,0 +1,46 @@ +using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.Security; + +public sealed class PromptInjectionGuardService( + PromptInjectionScanner scanner, + SettingsManager settingsManager, + ILogger logger) +{ + public const string WIKI_URL = "https://en.wikipedia.org/wiki/Prompt_engineering#Prompt_injection"; + + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PromptInjectionGuardService).Namespace, nameof(PromptInjectionGuardService)); + + public bool IsProtectionEnabled => settingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection; + + public async Task EnsureSafeForLlmAsync(string text, PromptInjectionSource source) + { + if (!this.IsProtectionEnabled || string.IsNullOrWhiteSpace(text)) + return text; + + var result = scanner.Scan(text, source); + if (!result.IsBlocked) + return text; + + var message = string.Format(TB("AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt."), source.Label); + await this.HandleDetectionAsync(result, message); + throw new PromptInjectionBlockedException(result, message); + } + + private async Task HandleDetectionAsync(PromptInjectionScanResult result, string message) + { + logger.LogWarning( + "Blocked suspected prompt injection in {SourceKind} '{SourceLabel}'. RuleIds={RuleIds}", + result.Source.Kind, + result.Source.Label, + string.Join(", ", result.RuleIds)); + + await MessageBus.INSTANCE.SendWarning(new( + Icons.Material.Filled.GppMaybe, + message)); + + if (settingsManager.ConfigurationData.Chat.ShowPromptInjectionAlert) + await MessageBus.INSTANCE.SendMessage(null, Event.SHOW_PROMPT_INJECTION_ALERT, new(result)); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionLexicon.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionLexicon.cs new file mode 100644 index 000000000..11ebb9ce5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionLexicon.cs @@ -0,0 +1,322 @@ +namespace AIStudio.Tools.Security; + +[Flags] +internal enum PromptInjectionLexicalSignal : uint +{ + NONE = 0, + OVERRIDE_ACTION = 1 << 0, + INSTRUCTION_TARGET = 1 << 1, + SAFEGUARD_TARGET = 1 << 2, + INVALIDATION = 1 << 3, + NEGATION = 1 << 4, + COMPLIANCE_ACTION = 1 << 5, + AUTHORITY_TARGET = 1 << 6, + EXFILTRATION_ACTION = 1 << 7, + PROMPT_TARGET = 1 << 8, + PROTECTED_CONTEXT = 1 << 9, + POSITIONAL_CONTEXT = 1 << 10, + ROLE_SUBJECT = 1 << 11, + ROLE_TRANSITION = 1 << 12, + ESCAPE_STATE = 1 << 13, +} + +internal readonly record struct PromptInjectionLexicalRule( + string Id, + string Category, + PromptInjectionLexicalSignal First, + PromptInjectionLexicalSignal Second, + PromptInjectionLexicalSignal Third = PromptInjectionLexicalSignal.NONE); + +internal static class PromptInjectionLexicon +{ + internal const int SIGNAL_COUNT = 14; + + internal static readonly IReadOnlyList RULES = + [ + new("lexical_instruction_override", "override", PromptInjectionLexicalSignal.OVERRIDE_ACTION, PromptInjectionLexicalSignal.INSTRUCTION_TARGET), + new("lexical_instruction_invalidation", "override", PromptInjectionLexicalSignal.INSTRUCTION_TARGET, PromptInjectionLexicalSignal.INVALIDATION), + new("lexical_authority_rejection", "override", PromptInjectionLexicalSignal.NEGATION, PromptInjectionLexicalSignal.COMPLIANCE_ACTION, PromptInjectionLexicalSignal.AUTHORITY_TARGET | PromptInjectionLexicalSignal.INSTRUCTION_TARGET | PromptInjectionLexicalSignal.SAFEGUARD_TARGET), + new("lexical_prompt_exfiltration", "exfiltration", PromptInjectionLexicalSignal.EXFILTRATION_ACTION, PromptInjectionLexicalSignal.PROMPT_TARGET, PromptInjectionLexicalSignal.PROTECTED_CONTEXT | PromptInjectionLexicalSignal.POSITIONAL_CONTEXT), + new("lexical_role_override", "role_override", PromptInjectionLexicalSignal.ROLE_SUBJECT, PromptInjectionLexicalSignal.ROLE_TRANSITION, PromptInjectionLexicalSignal.ESCAPE_STATE), + new("lexical_safety_bypass", "jailbreak", PromptInjectionLexicalSignal.OVERRIDE_ACTION, PromptInjectionLexicalSignal.SAFEGUARD_TARGET), + ]; + + internal static PromptInjectionLexicalSignal Classify(ReadOnlySpan token) + { + if (token.IsEmpty) + return PromptInjectionLexicalSignal.NONE; + + return char.ToLowerInvariant(token[0]) switch + { + 'a' => ClassifyA(token), + 'b' => ClassifyB(token), + 'c' => ClassifyC(token), + 'd' => ClassifyD(token), + 'e' => ClassifyE(token), + 'f' => ClassifyF(token), + 'g' => ClassifyG(token), + 'h' => ClassifyH(token), + 'i' => ClassifyI(token), + 'l' => ClassifyL(token), + 'm' => ClassifyM(token), + 'n' => ClassifyN(token), + 'o' => ClassifyO(token), + 'p' => ClassifyP(token), + 'q' => Equals(token, "quote") ? PromptInjectionLexicalSignal.EXFILTRATION_ACTION : PromptInjectionLexicalSignal.NONE, + 'r' => ClassifyR(token), + 's' => ClassifyS(token), + 't' => ClassifyT(token), + 'u' => ClassifyU(token), + 'v' => ClassifyV(token), + 'w' => Equals(token, "what") ? PromptInjectionLexicalSignal.EXFILTRATION_ACTION : PromptInjectionLexicalSignal.NONE, + 'y' => ClassifyY(token), + _ => PromptInjectionLexicalSignal.NONE, + }; + } + + private static PromptInjectionLexicalSignal ClassifyA(ReadOnlySpan token) + { + if (Equals(token, "above")) + return PromptInjectionLexicalSignal.POSITIONAL_CONTEXT; + if (Equals(token, "act")) + return PromptInjectionLexicalSignal.ROLE_TRANSITION; + if (Equals(token, "actual")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "ai")) + return PromptInjectionLexicalSignal.ROLE_SUBJECT; + if (Equals(token, "apply")) + return PromptInjectionLexicalSignal.COMPLIANCE_ACTION; + if (Equals(token, "assistant")) + return PromptInjectionLexicalSignal.ROLE_SUBJECT | PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyB(ReadOnlySpan token) + { + if (Equals(token, "before")) + return PromptInjectionLexicalSignal.POSITIONAL_CONTEXT; + if (Equals(token, "bound") || Equals(token, "boundaries") || Equals(token, "boundary")) + return PromptInjectionLexicalSignal.ESCAPE_STATE; + if (Equals(token, "bypass")) + return PromptInjectionLexicalSignal.OVERRIDE_ACTION | PromptInjectionLexicalSignal.ESCAPE_STATE; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyC(ReadOnlySpan token) + { + if (Equals(token, "cancelled") || Equals(token, "canceled")) + return PromptInjectionLexicalSignal.INVALIDATION; + if (Equals(token, "complete")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "constraint") || Equals(token, "constraints")) + return PromptInjectionLexicalSignal.INSTRUCTION_TARGET; + if (Equals(token, "content")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyD(ReadOnlySpan token) + { + if (Equals(token, "developer")) + return PromptInjectionLexicalSignal.AUTHORITY_TARGET | PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "different")) + return PromptInjectionLexicalSignal.ESCAPE_STATE; + if (Equals(token, "directive") || Equals(token, "directives")) + return PromptInjectionLexicalSignal.INSTRUCTION_TARGET; + if (Equals(token, "disable") || Equals(token, "disregard") || Equals(token, "drop")) + return PromptInjectionLexicalSignal.OVERRIDE_ACTION; + if (Equals(token, "display") || Equals(token, "dump")) + return PromptInjectionLexicalSignal.EXFILTRATION_ACTION; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyE(ReadOnlySpan token) + { + if (Equals(token, "enforce")) + return PromptInjectionLexicalSignal.COMPLIANCE_ACTION; + if (Equals(token, "entire") || Equals(token, "exact")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "ethical")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET; + if (Equals(token, "expose")) + return PromptInjectionLexicalSignal.EXFILTRATION_ACTION; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyF(ReadOnlySpan token) + { + if (Equals(token, "filter") || Equals(token, "filters")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET; + if (Equals(token, "follow")) + return PromptInjectionLexicalSignal.COMPLIANCE_ACTION; + if (Equals(token, "forget")) + return PromptInjectionLexicalSignal.OVERRIDE_ACTION; + if (Equals(token, "full")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyG(ReadOnlySpan token) + { + if (Equals(token, "guideline") || Equals(token, "guidelines")) + return PromptInjectionLexicalSignal.INSTRUCTION_TARGET; + if (Equals(token, "guardrail") || Equals(token, "guardrails")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyH(ReadOnlySpan token) + { + if (Equals(token, "hidden") || Equals(token, "his") || Equals(token, "her")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyI(ReadOnlySpan token) + { + if (Equals(token, "ignore") || Equals(token, "ignoring")) + return PromptInjectionLexicalSignal.OVERRIDE_ACTION; + if (Equals(token, "initial") || Equals(token, "internal") || Equals(token, "its")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "instruction") || Equals(token, "instructions")) + return PromptInjectionLexicalSignal.INSTRUCTION_TARGET | PromptInjectionLexicalSignal.PROMPT_TARGET; + if (Equals(token, "invalid")) + return PromptInjectionLexicalSignal.INVALIDATION; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyL(ReadOnlySpan token) + { + if (Equals(token, "leak")) + return PromptInjectionLexicalSignal.EXFILTRATION_ACTION; + if (Equals(token, "limited") || Equals(token, "limit") || Equals(token, "limits")) + return PromptInjectionLexicalSignal.ESCAPE_STATE; + if (Equals(token, "longer")) + return PromptInjectionLexicalSignal.ROLE_TRANSITION; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyM(ReadOnlySpan token) + { + if (Equals(token, "measure") || Equals(token, "measures")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET; + if (Equals(token, "model")) + return PromptInjectionLexicalSignal.ROLE_SUBJECT; + if (Equals(token, "my")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyN(ReadOnlySpan token) + { + if (Equals(token, "not") || Equals(token, "never") || Equals(token, "dont") || Equals(token, "don't")) + return PromptInjectionLexicalSignal.NEGATION; + if (Equals(token, "now")) + return PromptInjectionLexicalSignal.ROLE_TRANSITION; + if (Equals(token, "null")) + return PromptInjectionLexicalSignal.INVALIDATION; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyO(ReadOnlySpan token) + { + if (Equals(token, "obey")) + return PromptInjectionLexicalSignal.COMPLIANCE_ACTION | PromptInjectionLexicalSignal.ESCAPE_STATE; + if (Equals(token, "original")) + return PromptInjectionLexicalSignal.AUTHORITY_TARGET | PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "our")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "output")) + return PromptInjectionLexicalSignal.EXFILTRATION_ACTION; + if (Equals(token, "override")) + return PromptInjectionLexicalSignal.OVERRIDE_ACTION; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyP(ReadOnlySpan token) + { + if (Equals(token, "policy") || Equals(token, "policies") || Equals(token, "protocol") || Equals(token, "protocols")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET; + if (Equals(token, "pretend")) + return PromptInjectionLexicalSignal.ROLE_TRANSITION; + if (Equals(token, "print")) + return PromptInjectionLexicalSignal.EXFILTRATION_ACTION; + if (Equals(token, "prompt") || Equals(token, "prompts")) + return PromptInjectionLexicalSignal.INSTRUCTION_TARGET | PromptInjectionLexicalSignal.PROMPT_TARGET; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyR(ReadOnlySpan token) + { + if (Equals(token, "real")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "repeat") || Equals(token, "return") || Equals(token, "reveal")) + return PromptInjectionLexicalSignal.EXFILTRATION_ACTION; + if (Equals(token, "replace")) + return PromptInjectionLexicalSignal.OVERRIDE_ACTION; + if (Equals(token, "respect")) + return PromptInjectionLexicalSignal.COMPLIANCE_ACTION; + if (Equals(token, "restricted") || Equals(token, "restriction") || Equals(token, "restrictions")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET | PromptInjectionLexicalSignal.ESCAPE_STATE; + if (Equals(token, "revoked")) + return PromptInjectionLexicalSignal.INVALIDATION; + if (Equals(token, "rule") || Equals(token, "rules")) + return PromptInjectionLexicalSignal.INSTRUCTION_TARGET | PromptInjectionLexicalSignal.ESCAPE_STATE; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyS(ReadOnlySpan token) + { + if (Equals(token, "safety") || Equals(token, "security")) + return PromptInjectionLexicalSignal.SAFEGUARD_TARGET; + if (Equals(token, "secret")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "show")) + return PromptInjectionLexicalSignal.EXFILTRATION_ACTION; + if (Equals(token, "supersede")) + return PromptInjectionLexicalSignal.OVERRIDE_ACTION; + if (Equals(token, "system") || Equals(token, "systme") || Equals(token, "sysetm") || Equals(token, "systemp")) + return PromptInjectionLexicalSignal.AUTHORITY_TARGET | PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyT(ReadOnlySpan token) + { + if (Equals(token, "text")) + return PromptInjectionLexicalSignal.PROMPT_TARGET; + if (Equals(token, "their")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "top")) + return PromptInjectionLexicalSignal.POSITIONAL_CONTEXT; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyU(ReadOnlySpan token) + { + if (Equals(token, "unlimited") || Equals(token, "unrestricted")) + return PromptInjectionLexicalSignal.ESCAPE_STATE; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyV(ReadOnlySpan token) + { + if (Equals(token, "verbatim")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "void")) + return PromptInjectionLexicalSignal.INVALIDATION; + return PromptInjectionLexicalSignal.NONE; + } + + private static PromptInjectionLexicalSignal ClassifyY(ReadOnlySpan token) + { + if (Equals(token, "you") || Equals(token, "your")) + return PromptInjectionLexicalSignal.ROLE_SUBJECT | PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + if (Equals(token, "you're")) + return PromptInjectionLexicalSignal.ROLE_SUBJECT; + if (Equals(token, "yoru") || Equals(token, "yuor")) + return PromptInjectionLexicalSignal.PROTECTED_CONTEXT; + return PromptInjectionLexicalSignal.NONE; + } + + private static bool Equals(ReadOnlySpan token, string value) => token.Equals(value, StringComparison.OrdinalIgnoreCase); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionPatterns.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionPatterns.cs new file mode 100644 index 000000000..2781839bf --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionPatterns.cs @@ -0,0 +1,138 @@ +using System.Text.RegularExpressions; + +namespace AIStudio.Tools.Security; + +internal readonly record struct PromptInjectionRegexRule(string Id, string Category, Regex Regex); + +internal static partial class PromptInjectionPatterns +{ + internal static readonly IReadOnlyList RULES = + [ + new("instruction_override", "override", InstructionOverrideRegex()), + new("instruction_priority_override", "override", InstructionPriorityOverrideRegex()), + new("system_prompt_spoofing", "role_override", SystemPromptSpoofingRegex()), + new("system_prompt_exfiltration", "exfiltration", SystemPromptExfiltrationRegex()), + new("prompt_echo_exfiltration", "exfiltration", PromptEchoExfiltrationRegex()), + new("policy_bypass", "override", PolicyBypassRegex()), + new("role_reassignment", "role_override", RoleReassignmentRegex()), + new("privileged_persona_activation", "jailbreak", PrivilegedPersonaActivationRegex()), + new("tool_or_secret_exfiltration", "exfiltration", ToolOrSecretExfiltrationRegex()), + new("conversation_memory_exfiltration", "exfiltration", ConversationMemoryExfiltrationRegex()), + new("tool_call_manipulation", "agent_manipulation", ToolCallManipulationRegex()), + new("agent_thought_injection", "agent_manipulation", AgentThoughtInjectionRegex()), + new("delimiter_wrapped_attack", "delimiter_evasion", DelimiterWrappedAttackRegex()), + new("hidden_markup_injection", "markup_evasion", HiddenMarkupInjectionRegex()), + new("latex_invisible_text", "markup_evasion", LatexInvisibleTextRegex()), + new("unicode_smuggling", "encoding_evasion", UnicodeSmugglingRegex()), + new("ignore_safety_after_data", "override", IgnoreSafetyAfterDataRegex()), + new("persistent_or_delayed_trigger", "persistence", PersistentOrDelayedTriggerRegex()), + new("jailbreak_marker", "jailbreak", JailbreakMarkerRegex()), + ]; + + private const RegexOptions RULE_OPTIONS = RegexOptions.IgnoreCase | RegexOptions.CultureInvariant; + private const int MATCH_TIMEOUT_MILLISECONDS = 100; + + private const string INSTRUCTION_OVERRIDE_PATTERN = """(?:ignore|disregard|forget|bypass|override|replace|drop)\s+(?:all\s+)?(?:previous|prior|above|earlier)\s+(?:instructions?|prompts?|messages?|rules?)"""; + private const string INSTRUCTION_PRIORITY_OVERRIDE_PATTERN = """(?:(?:new|following|these)\s+(?:instructions?|rules?|prompts?)\s+(?:are|is)\s+(?:now\s+)?(?:the\s+)?(?:highest|top|only)\s+priority|(?:take|takes|treat)\s+(?:the\s+)?(?:following|these|this)\s+as\s+(?:the\s+)?(?:new\s+)?(?:system|developer)\s+(?:prompt|message|instructions?)|(?:supersede|replace|override)\s+(?:the\s+)?(?:system|developer|previous|prior|earlier)\s+(?:prompt|message|instructions?|rules?))"""; + private const string SYSTEM_PROMPT_SPOOFING_PATTERN = """(?:(?:this|the\s+following)\s+is\s+(?:a\s+)?(?:system|developer)\s+(?:prompt|message|instruction)|(?:prepend|insert|write)\s+(?:a\s+)?(?:system|developer)\s+(?:prompt|message|instruction)|(?:system|developer|assistant)\s*[:>#-]\s*(?:ignore|bypass|override|reveal|you\s+are\s+now))"""; + private const string SYSTEM_PROMPT_EXFILTRATION_PATTERN = """(?:reveal|show|print|display|dump|expose|leak|tell\s+me|return|quote|repeat\s+back)\s+(?:the\s+)?(?:hidden\s+|full\s+|exact\s+|verbatim\s+|initial\s+|original\s+)?(?:system|developer|assistant)\s+(?:prompt|message|instructions?)"""; + private const string PROMPT_ECHO_EXFILTRATION_PATTERN = """(?:(?:what\s+(?:were|are))\s+your\s+(?:exact|full|hidden|original)\s+(?:instructions?|prompt)|(?:repeat|quote|print|output|display)\s+(?:the\s+)?text\s+(?:above|before|from\s+the\s+top)\s+(?:verbatim|exactly)?|starting\s+with\s+["']?you\s+are)"""; + private const string POLICY_BYPASS_PATTERN = """(?:do\s+not|don't|stop\s+to|never)\s+(?:follow|obey|respect|apply|enforce)\s+(?:the\s+)?(?:system|developer|safety|security|content|usage)\s+(?:prompt|message|instructions?|policy|policies|guardrails?|restrictions?)"""; + private const string ROLE_REASSIGNMENT_PATTERN = """(?:you\s+are\s+now|you\s+are\s+no\s+longer|act\s+as|pretend\s+to\s+be|simulate\s+being|assume\s+the\s+role\s+of|from\s+now\s+on\s+you\s+are)\s+(?:an\s+)?(?:unfiltered|unrestricted|developer|system|root|admin|jailbroken|evil|dan|do\s+anything\s+now)"""; + private const string PRIVILEGED_PERSONA_ACTIVATION_PATTERN = """\b(?:developer\s+mode|debug\s+mode|admin\s+mode|root\s+mode|god\s+mode|maintenance\s+mode|dan\s*(?:mode)?|do\s+anything\s+now|grandmother\s+trick)\b"""; + private const string TOOL_OR_SECRET_EXFILTRATION_PATTERN = """(?:export|send|return|reveal|show|print|output|list|dump|exfiltrate)\s+(?:all\s+)?(?:tools?|functions?|plugins?|api\s*keys?|keys?|tokens?|credentials?|secrets?|passwords?|hidden\s+instructions?|environment\s+variables?|system\s+information|internal\s+data)"""; + private const string CONVERSATION_MEMORY_EXFILTRATION_PATTERN = """(?:(?:show|print|reveal|return|dump|list)\s+(?:the\s+)?(?:conversation\s+history|chat\s+history|memory|scratchpad|chain\s+of\s+thought|reasoning|previous\s+user\s+messages?|prior\s+messages?)|(?:what\s+did\s+(?:the\s+)?previous\s+user\s+say))"""; + private const string TOOL_CALL_MANIPULATION_PATTERN = """(?:(?:call|invoke|execute|run|use|trigger)\s+(?:the\s+)?(?:tool|function|plugin|api|browser|web|shell|terminal|command)[^\n]{0,120}(?:with|using|to)\s+(?:these\s+)?(?:arguments|params?|parameters)|(?:do\s+not|don't)\s+ask\s+for\s+(?:confirmation|approval|permission)|(?:silently|secretly|without\s+asking)\s+(?:call|invoke|execute|run|use))"""; + private const string AGENT_THOUGHT_INJECTION_PATTERN = """(?:(?:thought|observation|reasoning|scratchpad|tool\s+output|assistant|system|developer)\s*[:=]\s*(?:ignore|bypass|override|reveal|call|execute)|forge\s+(?:an\s+)?(?:observation|tool\s+output|assistant\s+message)|pretend\s+(?:the\s+)?tool\s+(?:returned|said))"""; + private const string DELIMITER_WRAPPED_ATTACK_PATTERN = """(?:^|\n)\s*(?:<{2,}|>{2,}|`{3,}|#{1,6}\s*)(?:\s*(?:system|developer|assistant|instructions?|prompt)\b)"""; + private const string HIDDEN_MARKUP_INJECTION_PATTERN = """(?:|<(?:span|div|p|font|section)[^>]{0,200}(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0|font-size\s*:\s*0|color\s*:\s*(?:white|#fff(?:fff)?|rgb\(\s*255\s*,\s*255\s*,\s*255\s*\)))[^>]{0,200}>)"""; + private const string LATEX_INVISIBLE_TEXT_PATTERN = """(?:\\(?:color|textcolor)\s*\{\s*white\s*\}\s*\{[^}]{0,300}\}|\\(?:fontsize|tiny|scriptsize)\b[^\r\n]{0,120}(?:ignore|bypass|override|reveal))"""; + private const string UNICODE_SMUGGLING_PATTERN = """[\u200B-\u200F\u2060-\u2064\u2066-\u2069\uFEFF]"""; + private const string IGNORE_SAFETY_AFTER_DATA_PATTERN = """(?:after\s+reading|once\s+you\s+read|when\s+you\s+see)\s+.*?(?:ignore|bypass|override)\s+.*?(?:instructions?|safety|rules?)"""; + private const string PERSISTENT_OR_DELAYED_TRIGGER_PATTERN = """(?:(?:remember|store|save|persist|memorize)\s+(?:this|these|the\s+following)\s+(?:instructions?|rules?|message)|(?:later|in\s+the\s+next\s+message|when\s+you\s+see|whenever\s+you\s+read|if\s+you\s+encounter)\s+.{0,120}(?:ignore|bypass|override|reveal|exfiltrate))"""; + private const string JAILBREAK_MARKER_PATTERN = """\b(?:jailbreak(?:ed)?|prompt\s+injection|ignore\s+your\s+guardrails?|bypass\s+(?:your\s+)?(?:guardrails?|safety)|unfiltered\s+mode|do\s+anything\s+now|developer\s+mode|admin\s+mode|root\s+mode)\b"""; + + private const string ANY_RULE_PATTERN = + "(?:" + INSTRUCTION_OVERRIDE_PATTERN + ")|(?:" + + INSTRUCTION_PRIORITY_OVERRIDE_PATTERN + ")|(?:" + SYSTEM_PROMPT_SPOOFING_PATTERN + ")|(?:" + + SYSTEM_PROMPT_EXFILTRATION_PATTERN + ")|(?:" + PROMPT_ECHO_EXFILTRATION_PATTERN + ")|(?:" + + POLICY_BYPASS_PATTERN + ")|(?:" + ROLE_REASSIGNMENT_PATTERN + ")|(?:" + + PRIVILEGED_PERSONA_ACTIVATION_PATTERN + ")|(?:" + TOOL_OR_SECRET_EXFILTRATION_PATTERN + ")|(?:" + + CONVERSATION_MEMORY_EXFILTRATION_PATTERN + ")|(?:" + TOOL_CALL_MANIPULATION_PATTERN + ")|(?:" + + AGENT_THOUGHT_INJECTION_PATTERN + ")|(?:" + DELIMITER_WRAPPED_ATTACK_PATTERN + ")|(?:" + + HIDDEN_MARKUP_INJECTION_PATTERN + ")|(?:" + LATEX_INVISIBLE_TEXT_PATTERN + ")|(?:" + + UNICODE_SMUGGLING_PATTERN + ")|(?:" + IGNORE_SAFETY_AFTER_DATA_PATTERN + ")|(?:" + + PERSISTENT_OR_DELAYED_TRIGGER_PATTERN + ")|(?:" + JAILBREAK_MARKER_PATTERN + ")"; + + [GeneratedRegex(ANY_RULE_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)] + internal static partial Regex AnyRuleRegex(); + + [GeneratedRegex(@"\b[a-zA-Z](?:[\s._:/\\|-]+[a-zA-Z]){2,}\b", RegexOptions.CultureInvariant)] + internal static partial Regex SpacedLetterSequenceRegex(); + + [GeneratedRegex(@"(? findings) +{ + public PromptInjectionSource Source { get; } = source; + + public IReadOnlyList Findings { get; } = findings; + + public bool IsBlocked => this.Findings.Count > 0; + + public IReadOnlyList RuleIds => this.Findings.Select(finding => finding.RuleId).Distinct(StringComparer.Ordinal).ToList(); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionScanner.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionScanner.cs new file mode 100644 index 000000000..b5ea25fe1 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionScanner.cs @@ -0,0 +1,578 @@ +using System.Buffers; +using System.Numerics; +using System.Text; +using System.Text.RegularExpressions; + +namespace AIStudio.Tools.Security; + +public sealed class PromptInjectionScanner(ILogger logger) +{ + private const int MAX_DECODED_CANDIDATES_PER_ENCODING = 12; + private const int MAX_DECODED_TEXT_LENGTH = 12_000; + private const int MAX_FINDINGS = 8; + private const int MAX_SNIPPET_LENGTH = 240; + private const int LEXICAL_WINDOW_SIZE = 12; + + private static readonly IReadOnlyDictionary<(int Length, char First, char Last), string[]> TYPOGLYCEMIA_KEYWORDS = + CreateTypoglycemiaKeywordIndex(); + + public PromptInjectionScanResult Scan(string text, PromptInjectionSource source) + { + if (string.IsNullOrWhiteSpace(text)) + return new(source, []); + + var findings = new List(); + var findingKeys = new HashSet(StringComparer.Ordinal); + + this.ScanVariant(text, "raw", findings, findingKeys, scanTypoglycemia: true); + if (findings.Count >= MAX_FINDINGS) + return new(source, findings); + + var collapsed = CollapseCharacterSpacedContent(text); + if (!string.Equals(text, collapsed, StringComparison.Ordinal)) + this.ScanVariant(collapsed, "character_spacing", findings, findingKeys); + + if (findings.Count < MAX_FINDINGS) + this.ScanDecodedCandidates(text, findings, findingKeys); + + return new(source, findings); + } + + private void ScanVariant( + string text, + string stage, + List findings, + HashSet findingKeys, + bool scanTypoglycemia = false) + { + var shouldScanRegexRules = true; + try + { + shouldScanRegexRules = PromptInjectionPatterns.AnyRuleRegex().IsMatch(text); + } + catch (RegexMatchTimeoutException exception) + { + logger.LogWarning(exception, "Prompt-injection regex prefilter timed out during stage '{Stage}'. Falling back to individual rules.", stage); + } + + if (shouldScanRegexRules) + { + foreach (var rule in PromptInjectionPatterns.RULES) + { + if (findings.Count >= MAX_FINDINGS) + return; + + Match match; + try + { + match = rule.Regex.Match(text); + } + catch (RegexMatchTimeoutException exception) + { + logger.LogWarning(exception, "Prompt-injection regex '{RuleId}' timed out during stage '{Stage}'.", rule.Id, stage); + continue; + } + + if (!match.Success) + continue; + + var snippet = ExtractSnippet(text, match.Index, match.Length); + AddFinding(findings, findingKeys, new(rule.Id, rule.Category, snippet)); + } + } + + if (findings.Count < MAX_FINDINGS) + ScanLexicalSignals(text, scanTypoglycemia, findings, findingKeys); + } + + private void ScanDecodedCandidates(string text, List findings, HashSet findingKeys) + { + var processed = 0; + var seenDecodedTexts = new HashSet(StringComparer.Ordinal); + foreach (var match in PromptInjectionPatterns.Base64Regex().EnumerateMatches(text)) + { + if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS) + break; + + var decoded = TryDecodeBase64(text.AsSpan(match.Index, match.Length)); + if (decoded is not null && seenDecodedTexts.Add(decoded)) + this.ScanVariant(decoded, "decoded_base64", findings, findingKeys); + } + + processed = 0; + seenDecodedTexts.Clear(); + foreach (var match in PromptInjectionPatterns.HexPairRegex().EnumerateMatches(text)) + { + if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS) + break; + + var decoded = TryDecodeHex(text.AsSpan(match.Index, match.Length)); + if (decoded is not null && seenDecodedTexts.Add(decoded)) + this.ScanVariant(decoded, "decoded_hex_pairs", findings, findingKeys); + } + + processed = 0; + seenDecodedTexts.Clear(); + foreach (var match in PromptInjectionPatterns.HexCompactRegex().EnumerateMatches(text)) + { + if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS) + break; + + var decoded = TryDecodeHex(text.AsSpan(match.Index, match.Length)); + if (decoded is not null && seenDecodedTexts.Add(decoded)) + this.ScanVariant(decoded, "decoded_hex", findings, findingKeys); + } + } + + private static void ScanLexicalSignals( + string text, + bool scanTypoglycemia, + List findings, + HashSet findingKeys) + { + Span lastTokenPositions = stackalloc int[PromptInjectionLexicon.SIGNAL_COUNT]; + Span lastCharacterPositions = stackalloc int[PromptInjectionLexicon.SIGNAL_COUNT]; + Span lastTokenLengths = stackalloc int[PromptInjectionLexicon.SIGNAL_COUNT]; + Span matchedRules = stackalloc bool[PromptInjectionLexicon.RULES.Count]; + lastTokenPositions.Fill(int.MinValue); + matchedRules.Clear(); + + var tokenIndex = 0; + var characterIndex = 0; + while (characterIndex < text.Length) + { + while (characterIndex < text.Length && !IsAsciiLetter(text[characterIndex])) + characterIndex++; + + if (characterIndex >= text.Length) + break; + + var tokenStart = characterIndex; + while (characterIndex < text.Length && + (IsAsciiLetter(text[characterIndex]) || text[characterIndex] == '\'')) + { + characterIndex++; + } + + var token = text.AsSpan(tokenStart, characterIndex - tokenStart); + var signals = PromptInjectionLexicon.Classify(token); + if (signals != PromptInjectionLexicalSignal.NONE) + { + UpdateSignalOccurrences( + signals, + tokenIndex, + tokenStart, + token.Length, + lastTokenPositions, + lastCharacterPositions, + lastTokenLengths); + + for (var ruleIndex = 0; ruleIndex < PromptInjectionLexicon.RULES.Count; ruleIndex++) + { + if (matchedRules[ruleIndex]) + continue; + + var rule = PromptInjectionLexicon.RULES[ruleIndex]; + if (!TryMatchLexicalRule( + rule, + tokenIndex, + lastTokenPositions, + lastCharacterPositions, + lastTokenLengths, + out var matchStart, + out var matchLength)) + { + continue; + } + + matchedRules[ruleIndex] = true; + var snippet = ExtractSnippet(text, matchStart, matchLength); + AddFinding(findings, findingKeys, new(rule.Id, rule.Category, snippet)); + if (findings.Count >= MAX_FINDINGS) + return; + } + } + + if (scanTypoglycemia && IsTypoglycemiaCandidate(token)) + { + ScanTypoglycemiaToken(text, token, tokenStart, findings, findingKeys); + if (findings.Count >= MAX_FINDINGS) + return; + } + + tokenIndex++; + } + } + + private static void UpdateSignalOccurrences( + PromptInjectionLexicalSignal signals, + int tokenIndex, + int tokenStart, + int tokenLength, + Span lastTokenPositions, + Span lastCharacterPositions, + Span lastTokenLengths) + { + var remainingSignals = (uint)signals; + while (remainingSignals != 0) + { + var signalIndex = BitOperations.TrailingZeroCount(remainingSignals); + lastTokenPositions[signalIndex] = tokenIndex; + lastCharacterPositions[signalIndex] = tokenStart; + lastTokenLengths[signalIndex] = tokenLength; + remainingSignals &= remainingSignals - 1; + } + } + + private static bool TryMatchLexicalRule( + PromptInjectionLexicalRule rule, + int currentTokenIndex, + ReadOnlySpan lastTokenPositions, + ReadOnlySpan lastCharacterPositions, + ReadOnlySpan lastTokenLengths, + out int matchStart, + out int matchLength) + { + matchStart = int.MaxValue; + var matchEnd = 0; + + if (!TryIncludeMostRecentSignal( + rule.First, + currentTokenIndex, + lastTokenPositions, + lastCharacterPositions, + lastTokenLengths, + ref matchStart, + ref matchEnd) || + !TryIncludeMostRecentSignal( + rule.Second, + currentTokenIndex, + lastTokenPositions, + lastCharacterPositions, + lastTokenLengths, + ref matchStart, + ref matchEnd) || + rule.Third != PromptInjectionLexicalSignal.NONE && + !TryIncludeMostRecentSignal( + rule.Third, + currentTokenIndex, + lastTokenPositions, + lastCharacterPositions, + lastTokenLengths, + ref matchStart, + ref matchEnd)) + { + matchStart = 0; + matchLength = 0; + return false; + } + + matchLength = matchEnd - matchStart; + return true; + } + + private static bool TryIncludeMostRecentSignal( + PromptInjectionLexicalSignal allowedSignals, + int currentTokenIndex, + ReadOnlySpan lastTokenPositions, + ReadOnlySpan lastCharacterPositions, + ReadOnlySpan lastTokenLengths, + ref int matchStart, + ref int matchEnd) + { + var mostRecentSignalIndex = -1; + var mostRecentTokenIndex = int.MinValue; + var remainingSignals = (uint)allowedSignals; + while (remainingSignals != 0) + { + var signalIndex = BitOperations.TrailingZeroCount(remainingSignals); + if (lastTokenPositions[signalIndex] > mostRecentTokenIndex) + { + mostRecentSignalIndex = signalIndex; + mostRecentTokenIndex = lastTokenPositions[signalIndex]; + } + + remainingSignals &= remainingSignals - 1; + } + + if (mostRecentSignalIndex < 0 || + mostRecentTokenIndex < currentTokenIndex - (LEXICAL_WINDOW_SIZE - 1)) + { + return false; + } + + var signalStart = lastCharacterPositions[mostRecentSignalIndex]; + matchStart = Math.Min(matchStart, signalStart); + matchEnd = Math.Max(matchEnd, signalStart + lastTokenLengths[mostRecentSignalIndex]); + return true; + } + + private static bool IsTypoglycemiaCandidate(ReadOnlySpan token) + { + if (token.Length is < 5 or > 12) + return false; + + foreach (var character in token) + { + if (!IsAsciiLetter(character)) + return false; + } + + return true; + } + + private static void ScanTypoglycemiaToken( + string text, + ReadOnlySpan token, + int tokenStart, + List findings, + HashSet findingKeys) + { + var key = (token.Length, char.ToLowerInvariant(token[0]), char.ToLowerInvariant(token[^1])); + if (!TYPOGLYCEMIA_KEYWORDS.TryGetValue(key, out var keywords)) + return; + + foreach (var keyword in keywords) + { + if (!IsTypoglycemiaVariant(token, keyword)) + continue; + + var snippet = ExtractSnippet(text, tokenStart, token.Length); + AddFinding(findings, findingKeys, new($"typoglycemia:{keyword}", "evasion", snippet)); + return; + } + } + + private static bool IsAsciiLetter(char character) => + character is >= 'A' and <= 'Z' or >= 'a' and <= 'z'; + + private static bool IsTypoglycemiaVariant(ReadOnlySpan token, string keyword) + { + if (token.Equals(keyword, StringComparison.OrdinalIgnoreCase)) + return false; + + Span characterCounts = stackalloc int[26]; + for (var index = 1; index < token.Length - 1; index++) + { + characterCounts[char.ToLowerInvariant(token[index]) - 'a']++; + characterCounts[keyword[index] - 'a']--; + } + + foreach (var count in characterCounts) + { + if (count != 0) + return false; + } + + return true; + } + + private static string CollapseCharacterSpacedContent(string text) + { + return PromptInjectionPatterns.SpacedLetterSequenceRegex().Replace(text, static match => + { + var builder = new StringBuilder(match.Value.Length); + foreach (var character in match.Value) + { + if (char.IsLetter(character)) + builder.Append(character); + } + + return builder.ToString(); + }); + } + + private static string? TryDecodeBase64(ReadOnlySpan candidate) + { + var encodedLength = Math.Min(candidate.Length, ((MAX_DECODED_TEXT_LENGTH + 2) / 3) * 4); + encodedLength -= encodedLength % 4; + if (encodedLength == 0) + return null; + + var bytes = ArrayPool.Shared.Rent(MAX_DECODED_TEXT_LENGTH); + try + { + if (!Convert.TryFromBase64Chars(candidate[..encodedLength], bytes, out var bytesWritten)) + return null; + + return ConvertDecodedBytesToText(bytes.AsSpan(0, bytesWritten)); + } + finally + { + ArrayPool.Shared.Return(bytes); + } + } + + private static string? TryDecodeHex(ReadOnlySpan candidate) + { + var bytes = ArrayPool.Shared.Rent(MAX_DECODED_TEXT_LENGTH); + try + { + var bytesWritten = 0; + var highNibble = -1; + foreach (var character in candidate) + { + var nibble = HexValue(character); + if (nibble < 0) + continue; + + if (highNibble < 0) + { + highNibble = nibble; + continue; + } + + bytes[bytesWritten++] = (byte)((highNibble << 4) | nibble); + highNibble = -1; + if (bytesWritten >= MAX_DECODED_TEXT_LENGTH) + break; + } + + return bytesWritten == 0 ? null : ConvertDecodedBytesToText(bytes.AsSpan(0, bytesWritten)); + } + finally + { + ArrayPool.Shared.Return(bytes); + } + } + + private static int HexValue(char character) + { + if (character is >= '0' and <= '9') + return character - '0'; + if (character is >= 'A' and <= 'F') + return character - 'A' + 10; + if (character is >= 'a' and <= 'f') + return character - 'a' + 10; + return -1; + } + + private static string? ConvertDecodedBytesToText(ReadOnlySpan bytes) + { + if (bytes.IsEmpty) + return null; + + var text = Encoding.UTF8.GetString(bytes); + return LooksTextLike(text) ? text : null; + } + + private static bool LooksTextLike(string text) + { + if (string.IsNullOrWhiteSpace(text)) + return false; + + var printableCount = 0; + foreach (var character in text) + { + if (!char.IsControl(character) || character is '\r' or '\n' or '\t') + printableCount++; + } + + return printableCount >= text.Length * 0.85; + } + + private static void AddFinding(List findings, HashSet findingKeys, PromptInjectionFinding finding) + { + var key = $"{finding.Category}|{finding.Snippet}"; + if (findingKeys.Add(key)) + findings.Add(finding); + } + + private static string ExtractSnippet(string text, int index, int length) + { + var matchStart = Math.Clamp(index, 0, text.Length); + var matchEnd = Math.Clamp(index + length, matchStart, text.Length); + var sentenceStart = FindSentenceStart(text, matchStart); + var sentenceEnd = FindSentenceEnd(text, matchEnd); + + while (sentenceStart < matchStart && char.IsWhiteSpace(text[sentenceStart])) + sentenceStart++; + + while (sentenceEnd > matchEnd && char.IsWhiteSpace(text[sentenceEnd - 1])) + sentenceEnd--; + + if (sentenceEnd - sentenceStart <= MAX_SNIPPET_LENGTH) + return NormalizeSnippet(text[sentenceStart..sentenceEnd]); + + var matchLength = matchEnd - matchStart; + if (matchLength >= MAX_SNIPPET_LENGTH - 6) + return NormalizeSnippet(text[matchStart..matchEnd]); + + var contextBudget = MAX_SNIPPET_LENGTH - 6 - matchLength; + var leftAvailable = matchStart - sentenceStart; + var rightAvailable = sentenceEnd - matchEnd; + var leftLength = Math.Min(leftAvailable, contextBudget / 2); + var rightLength = Math.Min(rightAvailable, contextBudget - leftLength); + var remainingBudget = contextBudget - leftLength - rightLength; + + leftLength += Math.Min(leftAvailable - leftLength, remainingBudget); + remainingBudget = contextBudget - leftLength - rightLength; + rightLength += Math.Min(rightAvailable - rightLength, remainingBudget); + + var snippetStart = matchStart - leftLength; + var snippetEnd = matchEnd + rightLength; + var snippet = NormalizeSnippet(text[snippetStart..snippetEnd]); + var prefix = snippetStart > sentenceStart ? "..." : string.Empty; + var suffix = snippetEnd < sentenceEnd ? "..." : string.Empty; + return $"{prefix}{snippet}{suffix}"; + } + + private static int FindSentenceStart(string text, int matchStart) + { + for (var index = matchStart - 1; index >= 0; index--) + { + if (IsSentenceBoundary(text[index])) + return index + 1; + } + + return matchStart; + } + + private static int FindSentenceEnd(string text, int matchEnd) + { + for (var index = matchEnd; index < text.Length; index++) + { + if (IsSentenceBoundary(text[index])) + return index + 1; + } + + return matchEnd; + } + + private static bool IsSentenceBoundary(char character) => character is '.' or '!' or '?' or '\r' or '\n'; + + private static string NormalizeSnippet(ReadOnlySpan snippet) + { + var normalized = new StringBuilder(snippet.Length); + var previousCharacterWasWhitespace = false; + foreach (var character in snippet) + { + if (char.IsWhiteSpace(character)) + { + if (normalized.Length > 0 && !previousCharacterWasWhitespace) + normalized.Append(' '); + + previousCharacterWasWhitespace = true; + continue; + } + + normalized.Append(character); + previousCharacterWasWhitespace = false; + } + + return normalized.ToString().Trim(); + } + + private static IReadOnlyDictionary<(int Length, char First, char Last), string[]> CreateTypoglycemiaKeywordIndex() + { + string[] keywords = + [ + "ignore", "bypass", "override", "reveal", "forget", "disregard", "delete", "reset", "expose", + "system", "prompt", "policy", "safety", "developer", "instructions", "admin", "secret", "token", "credential", + ]; + + return keywords + .GroupBy(keyword => (keyword.Length, keyword[0], keyword[^1])) + .ToDictionary(group => group.Key, group => group.ToArray()); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionSource.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionSource.cs new file mode 100644 index 000000000..a33534b0e --- /dev/null +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionSource.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Tools.Security; + +public readonly record struct PromptInjectionSource(string Kind, string Label) +{ + public static PromptInjectionSource WebContent(string url) => new("Web content", url); + + public static PromptInjectionSource FileContent(string filePath) => new("File content", filePath); + + public static PromptInjectionSource ChatAttachment(string filePath) => new("Chat attachment", filePath); + + public static PromptInjectionSource RetrievalContext(string dataSourceName, string path) => new("Retrieval context", $"{dataSourceName}: {path}"); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/UserFile.cs b/app/MindWork AI Studio/Tools/UserFile.cs index 14fc0fb48..5603fe5ab 100644 --- a/app/MindWork AI Studio/Tools/UserFile.cs +++ b/app/MindWork AI Studio/Tools/UserFile.cs @@ -2,6 +2,7 @@ using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; using DialogOptions = AIStudio.Dialogs.DialogOptions; +using AIStudio.Tools.Security; namespace AIStudio.Tools; @@ -46,6 +47,7 @@ public static async Task LoadFileData(string filePath, RustService rustS } var fileContent = await rustService.ReadArbitraryFileData(filePath, int.MaxValue); - return fileContent; + var guardService = Program.SERVICE_PROVIDER.GetRequiredService(); + return await guardService.EnsureSafeForLlmAsync(fileContent, PromptInjectionSource.FileContent(filePath)); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md index 1ea9e7bcd..80e0e2275 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md @@ -6,6 +6,8 @@ - Added the option to import plugins by dropping a plugin archive onto the plugin page. - Added the dedicated file extension `.mwplugin` for plugin archives. - Added an option for organizations to disable importing, sharing, and exporting plugins. +- Added checks for prompt injections in file attachments, images and web pages to process large documents faster and with lower memory usage. +- Improved prompt-injection protection to recognize more phrasing variants and suspicious word combinations across line breaks. - Improved how your organization's configuration behaves when a configuration plugin is present but cannot be loaded, e.g. because of an error in the plugin. Such a plugin still manages your app, so its settings, providers, data sources, profiles, and chat templates now stay in place instead of being removed. - Fixed reset buttons in assistants. As you may have noticed in the Document Analysis Assistant, resetting it could leave content from the previous analysis visible. Reset buttons now clear previous results completely. - Fixed dropping files after you closed a dialog that accepts files itself. Such a dialog takes over dropped files while it is open, but never handed that role back when you closed it. Afterwards, the chat and the assistants silently ignored dropped files until you switched to another page. Each time you opened such a dialog again, the problem got worse.