From 60bf84e457b9cbfb036d30bd42689f4220eeec7d Mon Sep 17 00:00:00 2001 From: "JJediny (merge automation)" Date: Sat, 3 Oct 2026 22:58:58 -0400 Subject: [PATCH] feat(alerts): add innocent security trigger runner, agent skill, and realtime IoC docs --- .../skills/security-alert-triggers/SKILL.md | 134 +++++ Taskfile.yml | 7 + docs/developing_realtime_ioc_alerts.md | 428 ++++++++++++++++ tools/trigger_alerts.py | 476 ++++++++++++++++++ 4 files changed, 1045 insertions(+) create mode 100644 .agents/skills/security-alert-triggers/SKILL.md create mode 100644 docs/developing_realtime_ioc_alerts.md create mode 100755 tools/trigger_alerts.py diff --git a/.agents/skills/security-alert-triggers/SKILL.md b/.agents/skills/security-alert-triggers/SKILL.md new file mode 100644 index 0000000..5268581 --- /dev/null +++ b/.agents/skills/security-alert-triggers/SKILL.md @@ -0,0 +1,134 @@ +--- +name: security-alert-triggers +description: Safe, innocent trigger commands and automated runner for security alerts across Falco, OSquery, ClamAV, RSigma, and AI Agent observability in LocalObserve. +tags: + - security + - alerts + - falco + - osquery + - clamav + - rsigma + - agent-observability + - openobserve + - mitre-attack + - desktop-notifications +--- + +# Security Alert Triggers Skill + +This skill provides a curated catalog of **innocent, non-destructive commands** to safely trigger, validate, and exercise all configured detection alerts across the LocalObserve monitoring stack without compromising host security or causing operational disruption. + +--- + +## 1. Quick Start + +Run the automated trigger runner to exercise all alert pipelines and dispatch native desktop notifications: + +```bash +# Trigger all alert streams and dispatch native Linux desktop notifications via notify-send +python3 tools/trigger_alerts.py --all + +# Or trigger an individual stream +python3 tools/trigger_alerts.py --stream falco +python3 tools/trigger_alerts.py --stream osquery +python3 tools/trigger_alerts.py --stream clamav +python3 tools/trigger_alerts.py --stream agent +python3 tools/trigger_alerts.py --stream rsigma +python3 tools/trigger_alerts.py --stream system-logs + +# Replay any queued alerts to your desktop screen without re-triggering +python3 tools/trigger_alerts.py --notify + +# Verify OpenObserve stream counts and alert dispatches only +python3 tools/trigger_alerts.py --verify +``` + +--- + +## 2. Desktop Notification Integration (`notify-send`) + +### Why the Host Notifier Bridge Is Required +Docker containers execute in an isolated namespace and do not have access to the user's host DBus session bus (`/run/user/1000/bus`) or Wayland/X11 display server. To bridge containerized alerts into native host desktop notifications: +1. The containerized `alert-receiver` writes alert JSON artifacts to `/var/log/alerts/` (bind-mounted to host `.data/alerts/`). +2. The host-side notifier ([`tools/desktop_notifier.py`](file:///home/john/LocalObserve/tools/desktop_notifier.py)) reads `.data/alerts/` and fires native `notify-send` popups with urgency styling, icons, and audio cues. + +### Useful Taskfile Commands for Desktop Alerting +```bash +# Replay all pending/queued alerts to screen once +task notify-desktop-replay + +# Run the persistent background daemon watching for new alerts in real time +task notify-desktop + +# Dismiss/clear stuck notification popups from screen +task notify-desktop-dismiss +``` + +--- + +## 3. Innocent Command Reference Matrix + +| Stream / Engine | Alert Rule Name | MITRE ATT&CK | Safe Innocent Trigger Command | Why It Is Safe & Innocent | Expected Log / Match Field | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **Falco** | `Falco-High-Severity` / `Read sensitive file untrusted` | T1555 (Credentials) | `python3 -c "open('/etc/security/pwquality.conf').read(10)"` | Read-only check of benign world-readable password quality configuration. No credentials exposed or modified. | `rule: "Read sensitive file untrusted"`, `priority: "Warning"` | +| **Falco** | `Falco-Shadow-Read` | T1003.008 (/etc/shadow) | `python3 -c "try: open('/etc/shadow')\nexcept Exception: pass"` | Unprivileged, non-root process triggers `openat` syscall on shadow path; fails safely with permission denied. | `rule: "Shadow File Read by Non-Auth Process"` or `rule: "Read sensitive file untrusted"` | +| **Falco** | `LD-PRELOAD-Injection` | T1574.006 (Hijack Execution) | `LD_PRELOAD=/lib/x86_64-linux-gnu/libc.so.6 id` | Preloads standard system GNU C library (already linked by the process). Executes standard benign `id`. | `rule: "Hijack Execution Flow with LD_PRELOAD"`, env contains `LD_PRELOAD=` | +| **Falco** | `Falco-Tmp-Execution` | T1059 (Execution) | `cp /bin/true /tmp/innocent_test_bin && /tmp/innocent_test_bin && rm /tmp/innocent_test_bin` | Copies GNU `true` (which immediately returns exit code 0) to `/tmp`, executes it, and removes it immediately. | `rule: "Execution from Temporary Directory"` | +| **OSquery** | `OSquery-FIM-Critical-File-Modified` | T1546.004 (Persistence) | `touch ~/.bashrc` | Updates the modification timestamp on existing benign user profile without modifying file contents. | `name: "file_events"`, `action: "UPDATED"` | +| **OSquery** | `OSquery-New-Cron-Job` | T1053.003 (Cron Persistence) | `(crontab -l 2>/dev/null; echo "# benign test: curl -s http://localhost:5080/healthz > /dev/null") \| crontab -` | Benign commented crontab entry referencing localhost health endpoint; cleans up via `crontab -r`. | `name: "crontab"`, `command LIKE '%curl%'` | +| **OSquery** | `OSquery-SUID-Binary-Added` | T1548.001 (SUID Abuse) | `docker exec localobserve-osquery-1 osqueryi --json "SELECT * FROM suid_bin LIMIT 5"` | Queries existing system SUID binaries via the container daemon without elevating permissions. | `name: "suid_bin_changes"` | +| **ClamAV** | `ClamAV-Malware-FOUND` | N/A (Standard AV Test) | `echo "/tmp/innocent_eicar_test.com: FOUND Win.Test.EICAR_HDB-1" >> .data/clamav/scan.log` | Uses the official EICAR antivirus test signature string designed specifically for safe AV alert testing. | `stream: "clamav"`, `scan_status: "FOUND"` | +| **AI Agent** | `AIAgent-Sensitive-File-Access` | T1003, T1565.001 | `uv run python tools/agent_session_demo.py --quick` | Scripted synthetic agent emits OTLP log telemetry requesting `/etc/shadow` tool call without altering the file. | `stream: "agent_logs"`, `gen_ai_tool_args LIKE '%/etc/shadow%'` | +| **AI Agent** | `AIAgent-Tool-Error-Spike` | T1059 (Execution Probe) | `uv run python tools/agent_session_demo.py --quick` | Emits a synthetic tool call error (`otel_status_code = "ERROR"`) simulating a failed connection. | `stream: "agent_logs"`, `otel_status_code = 'ERROR'` | +| **RSigma** | `Suspicious Namespace Unshare` | T1059, T1071 | `curl -s -X POST http://localhost:9090/api/v1/events -H "Content-Type: application/json" -d '{"name": "unshare", "cmdline": "unshare --user --map-root-user /bin/bash", "pid": 99001}'` | Direct HTTP event ingestion to RSigma streaming evaluator. No actual namespace unshare command is executed on the host. | RSigma webhook fires alert to `alert-receiver:9000` | +| **RSigma** | `AI Agent Dangerous Shell` | T1059, T1561.002 | `curl -s -X POST http://localhost:9090/api/v1/events -H "Content-Type: application/json" -d '{"service_namespace": "agent", "gen_ai_tool_name": "run_shell", "gen_ai_tool_args": "rm -rf /", "pid": 99002}'` | Harmless synthetic event posted to RSigma; tests detection of destructive commands without running any shell. | RSigma webhook fires `AI Agent Dangerous Shell Command` | +| **SystemLogs** | `SystemLogs-Auth-Failure-Spike` | T1110 (Brute Force) | `curl -s -u root@example.com:Complexpass#123 -X POST "http://localhost:5080/api/default/system_logs/_json" -d '[{"message": "sshd: Failed password for invalid user admin from 192.168.1.100 port 22"}]'` | Benign simulated authentication failure message ingested directly to OpenObserve log stream. | `stream: "system_logs"`, `message LIKE '%Failed password%'` | +| **SystemLogs** | `SystemLogs-Sudo-Abuse` | T1548.003 (Sudo Abuse) | `curl -s -u root@example.com:Complexpass#123 -X POST "http://localhost:5080/api/default/system_logs/_json" -d '[{"message": "sudo: guest : TTY=pts/1 ; PWD=/tmp ; USER=root ; COMMAND=/bin/sh"}]'` | Benign simulated sudo log entry for non-standard user ingested to OpenObserve log stream. | `stream: "system_logs"`, `message LIKE '%COMMAND=%'` | + +--- + +## 4. Verification Queries + +### OpenObserve SQL Queries + +Check hits across each stream via OpenObserve API: + +```bash +# 1. Falco Hits: +curl -u root@example.com:Complexpass#123 -X POST "http://localhost:5080/api/default/_search" \ + -H "Content-Type: application/json" \ + -d '{"query": {"sql": "SELECT rule, priority, output FROM \"falco\" ORDER BY _timestamp DESC LIMIT 5"}}' + +# 2. ClamAV Malware Hits: +curl -u root@example.com:Complexpass#123 -X POST "http://localhost:5080/api/default/_search" \ + -H "Content-Type: application/json" \ + -d '{"query": {"sql": "SELECT scan_path, scan_status, virus_name FROM \"clamav\" WHERE scan_status = '\''FOUND'\'' LIMIT 5"}}' + +# 3. AI Agent Sensitive File Access: +curl -u root@example.com:Complexpass#123 -X POST "http://localhost:5080/api/default/_search" \ + -H "Content-Type: application/json" \ + -d '{"query": {"sql": "SELECT gen_ai_tool_name, gen_ai_tool_args, otel_status_code FROM \"agent_logs\" LIMIT 5"}}' + +# 4. OSquery Events: +curl -u root@example.com:Complexpass#123 -X POST "http://localhost:5080/api/default/_search" \ + -H "Content-Type: application/json" \ + -d '{"query": {"sql": "SELECT name, action FROM \"osquery\" ORDER BY _timestamp DESC LIMIT 5"}}' +``` + +### Webhook Alert Receiver Verification + +Inspect dispatched alert JSON files received by `alert-receiver`: + +```bash +docker exec localobserve-alert-receiver-1 ls -la /var/log/alerts +# View the newest alert: +docker exec localobserve-alert-receiver-1 sh -c 'cat /var/log/alerts/$(ls -t /var/log/alerts | head -1)' +``` + +--- + +## 5. Related Documentation + +- [Developing Realtime Indicators of Compromise (IoC) Alerts Guide](file:///home/john/LocalObserve/docs/developing_realtime_ioc_alerts.md) +- [Alert Payload Schema & Action Playbook Specification](file:///home/john/LocalObserve/docs/alerting_payload_schema.md) +- [Multi-Runtime & Alerting Deployment Guide](file:///home/john/LocalObserve/docs/runtimes_alerting_and_resource_guide.md) diff --git a/Taskfile.yml b/Taskfile.yml index 80e09ba..bdf3e39 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -91,6 +91,13 @@ tasks: cmds: - bash ./tools/trigger-detections.sh {{.FILTER}} + trigger-alerts: + desc: Safely trigger innocent alerts across all pipelines (Falco, OSquery, ClamAV, Agent, RSigma) and notify desktop + vars: + STREAM: '{{.STREAM | default ""}}' + cmds: + - 'python3 tools/trigger_alerts.py {{if .STREAM}}--stream {{.STREAM}}{{else}}--all{{end}}' + ensure-threat-intel: desc: Ensure .data/threat-intel exists with a valid default fragment so falco can boot cmds: diff --git a/docs/developing_realtime_ioc_alerts.md b/docs/developing_realtime_ioc_alerts.md new file mode 100644 index 0000000..eac8c5e --- /dev/null +++ b/docs/developing_realtime_ioc_alerts.md @@ -0,0 +1,428 @@ +# Developing Alerts for Suspicious Real-Time Indicators of Compromise (IoCs) + +This guide documents the architecture, rule-authoring methodology, testing workflows, and best practices for developing real-time detection alerts across the LocalObserve security stack. It covers kernel-level syscall interception (**Falco**), host state and file integrity monitoring (**OSquery**), streaming Sigma rule matching (**RSigma**), malware and signature detection (**ClamAV**), AI Agent behavioral telemetry (**AI Agent Observability**), and log correlation analytics (**OpenObserve**). + +--- + +## 1. Real-Time Detection Architecture + +LocalObserve implements a multi-layer defense-in-depth detection pipeline designed to intercept Indicators of Compromise (IoCs) at varying abstraction layers with minimal latency and negligible host overhead. + +``` +┌─────────────────────────────────────────────────────────────────────────────────────────────┐ +│ TELEMETRY SOURCES │ +│ │ +│ ┌───────────────────────┐ ┌──────────────────────┐ ┌────────────────────────────────┐ │ +│ │ Kernel Syscalls │ │ Host State & FIM │ │ AI Agent Telemetry │ │ +│ │ (modern-eBPF probe) │ │ (Inotify / Tables) │ │ (OTLP Log Instrumentation) │ │ +│ │ [Falco] │ │ [OSquery] │ │ [agent-demo / SDK] │ │ +│ └───────────┬───────────┘ └──────────┬───────────┘ └───────────────┬────────────────┘ │ +└──────────────┼──────────────────────────┼───────────────────────────────┼───────────────────┘ + │ JSONL │ JSON Results │ OTLP HTTP (:4318) + ▼ ▼ ▼ +┌─────────────────────────────────────────────────────────────────────────────────────────────┐ +│ OPENTELEMETRY COLLECTOR CONTRIB │ +│ │ +│ • file_log receivers (tailing /var/log/{falco,osquery,clamav}) │ +│ • Memory limiter & Batch processors │ +│ • OTTL source-level volume reduction (filter/drop_osquery_inventory) │ +│ • Schema flattening & field normalization (transform/flatten) │ +└──────────────────────────────┬───────────────────────────────┬──────────────────────────────┘ + │ │ + ▼ ▼ +┌──────────────────────────────────────────────┐ ┌────────────────────────────────────────────┐ +│ STREAMING EVALUATION (SUB-SECOND) │ │ LOG AGGREGATION & SQL ANALYTICS │ +│ │ │ │ +│ rsigma Engine (:9090) │ │ OpenObserve Engine (:5080) │ +│ • In-memory Sigma YAML evaluation │ │ • Stream partitioning (falco, osquery, │ +│ • Zero-disk evaluation of streaming events │ │ agent_logs, clamav, system_logs) │ +│ • Immediate webhook emission upon match │ │ • Real-time & Scheduled SQL alert checks │ +│ │ │ • Threshold & sliding time window queries │ +└──────────────────────┬───────────────────────┘ └─────────────────────┬──────────────────────┘ + │ │ + │ Webhook JSON Payload │ Webhook JSON Payload + ▼ ▼ +┌─────────────────────────────────────────────────────────────────────────────────────────────┐ +│ ALERT RECEIVER & DISPATCH (:9000) │ +│ │ +│ • Destination: http://alert-receiver:9000/hooks/security-alert-open │ +│ • Handler script: /hooks/notify.sh │ +│ • Persistent alert artifact storage: /var/log/alerts/.json │ +│ • Desktop visual alert: notify-send --urgency=critical │ +│ • Integrations: Slack, Email, Generic SIEM Webhook, Automated Action Playbooks │ +└─────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 2. Detection Layer Comparison Matrix + +| Detection Engine | Layer / Focus | Telemetry Source | Typical Detection Latency | Best Suited For IoCs | +| :--- | :--- | :--- | :--- | :--- | +| **Falco (eBPF)** | Kernel / System Calls | Linux Kernel tracepoints (`sys_enter`, `sys_exit`) via `modern_ebpf` | `< 100 ms` | Dynamic linker hijacking (`LD_PRELOAD`), unprivileged `/etc/shadow` reads, interactive container shells, `/dev/shm` or `/tmp` code execution. | +| **RSigma** | Streaming Log Evaluation | HTTP `/api/v1/events` or OTel collector forwarder | `< 250 ms` | Fast matching of standardized Sigma rules over command lines, namespace unshares, process arguments, and AI agent tool requests. | +| **OSquery** | Host State & FIM | Kernel inotify + Linux audit / periodic SQL sweeps | Real-time for FIM; `1m–15m` for sweeps | File modification to monitored configs (`.bashrc`, SSH keys), persistence via crontab / systemd, SUID binary additions, user creation. | +| **OpenObserve Alerts** | Log Correlation & Aggregation | Indexed log streams (`falco`, `osquery`, `agent_logs`, `clamav`, `system_logs`) | `1m–15m` (configurable) | Threshold spikes (e.g. $\ge 5$ tool call errors in 5 min), cross-stream joins, aggregate brute-force detection, pipeline health baselines. | +| **ClamAV** | Antivirus & File Signatures | Filesystem scanner / clamd socket | On-scan completion | Malicious payload drops, webshell signatures, EICAR test verification, known malware hashes. | + +--- + +## 3. Developing Falco eBPF Rules + +Falco inspects system calls directly at the kernel boundary using the modern eBPF driver. Rules are written in YAML and loaded from `falco_rules.local.yaml` (container sidecar) and `/etc/falco/rules.d/` (host service). + +### Rule Anatomy + +```yaml +- rule: Hijack Execution Flow with LD_PRELOAD + desc: Detect LD_PRELOAD in process environment for dynamic linker hijacking. + condition: > + spawned_process + and proc.env contains "LD_PRELOAD=" + output: > + Process executed with LD_PRELOAD environment variable | + user=%user.name command=%proc.cmdline env=%proc.env container=%container.id + priority: WARNING + tags: [mitre_privilege_escalation, mitre_defense_evasion, T1574.006] +``` + +### Key Fields & Conventions + +1. **`rule`**: Unique, descriptive rule title. +2. **`condition`**: Logical expression evaluating Falco filter fields: + - `spawned_process`: Shorthand macro for `evt.type = execve and evt.dir = <`. + - `open_read`: Syscalls opening files for reading (`evt.type in (open, openat, openat2) and evt.is_open_read = true`). + - `proc.cmdline` / `proc.exepath`: Process command string and binary path. + - `proc.env`: Environment variables array. + - `container.id`: `host` for bare-metal processes; container hash for Docker/Podman workloads. +3. **`output`**: Human- and parser-readable template string using `%field.name` tokens. +4. **`priority`**: Severity ranking: `NOTICE`, `WARNING`, `CRITICAL`, `ALERT`, or `EMERGENCY`. + > [!IMPORTANT] + > Ensure the priority aligns with OpenObserve's `Falco-High-Severity` alert query, which monitors `priority IN ('Notice','Warning','Critical','Alert','Emergency')`. +5. **`tags`**: MITRE ATT&CK technique IDs (`T1574.006`), tactics (`mitre_defense_evasion`), and environmental scopes (`host`, `container`). + +### Best Practices for Falco Rules + +- **Use built-in macros**: Leverage macros like `sensitive_files`, `user_known_container_binaries`, and `shell_procs` to keep conditions concise. +- **Differentiate host vs container**: To prevent alert fatigue in build containers, scope host-only attacks with `container.id = host`. +- **Minimize expensive string searches**: Place fast checks (`evt.type = execve`) before substring scans (`proc.cmdline contains "..."`). + +--- + +## 4. Developing Sigma Rules for RSigma + +RSigma executes Sigma detection logic over streaming JSON events in memory without maintaining heavy database indexes. Rules reside in `rules/sigma/active_rules/*.yaml`. + +### Rule Anatomy + +```yaml +title: Suspicious Namespace Unshare Command +id: 718c5dbc-b1a3-419b-a329-e7721d294257 +status: experimental +description: Detects unshare command executed with --user flag, commonly used for container escapes or privilege escalation. +references: + - https://man7.org/linux/man-pages/man1/unshare.1.html +author: LocalObserve Team +date: 2026-07-26 +logsource: + category: process_creation + product: linux +detection: + selection: + name: unshare + cmdline|contains: + - '--user' + - '-U' + filter: + cmdline|contains: + - '--mount' + condition: selection and not filter +level: high +tags: + - attack.t1059 + - attack.t1071 + - attack.privilege_escalation +``` + +### Developing AI Agent Sigma Detections + +RSigma also processes AI Agent tool telemetry forwarded from `agent_logs`: + +```yaml +title: AI Agent Dangerous Shell Command +id: 5a1e0000-0000-4a91-a1a1-000000000001 +status: stable +description: Detects high-risk destructive shell commands initiated by an AI agent tool call. +logsource: + category: ai_agent + product: localobserve +detection: + selection: + gen_ai_tool_name: run_shell + gen_ai_tool_args|contains: + - 'rm -rf /' + - 'mkfs.' + - 'dd if=/dev/' + - ':(){ :|:& };:' + condition: selection +level: critical +tags: + - attack.t1059 + - attack.t1561.002 +``` + +### Field Transformation Rules + +When authoring Sigma rules for RSigma: +- The `logsource` identifies the pipeline schema. +- Condition modifiers like `|contains`, `|startswith`, and `|endswith` are evaluated directly against the incoming JSON keys. +- Matches instantly dispatch an HTTP POST request to `http://alert-receiver:9000/hooks/security-alert-open` containing rule details, MITRE tags, and the triggering payload. + +--- + +## 5. Developing OSquery Real-Time & Scheduled Queries + +OSquery exposes the underlying operating system as a relational SQLite database. Detection logic is defined in `osqueryd.conf`. + +### Event-Driven Tables (FIM) vs Scheduled Sweeps + +OSquery supports two detection paradigms: + +1. **Event-Based Tables (`file_events`, `process_events`)**: + Powered by inotify and the Linux audit subsystem. Changes are recorded continuously in real time. + ```json + "file_events": { + "query": "SELECT * FROM file_events;", + "interval": 300, + "description": "FIM events for profile and init scripts [T1546.004] [T1037.004] [T1070.003]", + "removed": false + } + ``` + Monitored file paths are configured under the `file_paths` section: + ```json + "file_paths": { + "bash_profiles": [ + "/home/%/.bashrc", + "/root/.bashrc", + "/etc/profile.d/%%" + ], + "ssh_keys": [ + "/home/%/.ssh/%%", + "/root/.ssh/%%" + ] + } + ``` + +2. **Differential Scheduled Queries (`crontab`, `suid_bin`, `systemd_units`)**: + OSquery periodically runs SQL queries and logs only added or removed records (`"action": "added"`): + ```json + "crontab": { + "query": "SELECT * FROM crontab;", + "interval": 900, + "description": "Scheduled cron jobs - detect persistence via cron [T1053.003]" + } + ``` + +### Tuning for System Performance and SSD Protection + +High-frequency inventory sweeps can cause excessive disk I/O and SSD wear: +- **Low-churn persistence tables** (`crontab`, `startup_items`, `systemd_units`): Poll every `300s` to `900s`. +- **High-churn inventory tables** (`listening_ports`, `processes`, `mounts`): Dropped at the OpenTelemetry Collector boundary via `filter/drop_osquery_inventory` unless explicitly needed for deep forensics. + +--- + +## 6. Developing OpenObserve Stream Alerts + +OpenObserve evaluates alerting conditions over incoming log streams using SQL queries. All alert configurations are version-controlled in `alerts/openobserve/alerts.json` and synchronized via `tools/oo-alerts.sh`. + +### Alert Object Schema + +```json +{ + "name": "Falco-Shadow-Read", + "stream_type": "logs", + "stream_name": "falco", + "is_real_time": false, + "query_condition": { + "conditions": null, + "sql": "SELECT * FROM \"falco\" WHERE rule = 'Shadow File Read by Non-Auth Process'", + "promql": "", + "type": "custom", + "aggregation": null + }, + "trigger_condition": { + "period": 1, + "operator": ">=", + "frequency": 1, + "threshold": 1, + "silence": 10, + "frequency_type": "minutes", + "timezone": "UTC" + }, + "destinations": [ + "localhost-webhook" + ], + "enabled": true, + "description": "Unauthorized access attempt to /etc/shadow [T1003.008]" +} +``` + +### Critical Stream Naming Rule + +> [!WARNING] +> OpenObserve automatically sanitizes hyphens (`-`) in stream headers to underscores (`_`). +> - An OTLP stream header `stream-name: agent-logs` creates the table `"agent_logs"`. +> - An OTLP stream header `stream-name: system-logs` creates the table `"system_logs"`. +> +> Always quote sanitized table names in your alert SQL queries (e.g. `SELECT * FROM "agent_logs"` or `SELECT * FROM "system_logs"`). + +### GitOps Synchronization + +Synchronize alert definitions between the repository and a running OpenObserve cluster: + +```bash +# 1. Register the alert-receiver webhook destination in OpenObserve +bash tools/oo-alerts.sh setup-destination + +# 2. Import all alert definitions from alerts/openobserve/alerts.json +bash tools/oo-alerts.sh import + +# 3. Export active alerts from OpenObserve back to Git +bash tools/oo-alerts.sh export + +# 4. Dispatch a test webhook to verify end-to-end receipt +bash tools/oo-alerts.sh test +``` + +--- + +## 7. Developing AI Agent Observability Alerts + +AI Agent tools that execute code, browse the filesystem, or interact with external APIs represent a unique threat surface. LocalObserve captures every agent tool call via OpenTelemetry structured logs. + +### Instrumenting Agent Tool Telemetry + +Using [`tools/agent_otel_prelude.py`](../tools/agent_otel_prelude.py): + +```python +import agent_otel_prelude as prelude + +# Emit tool call telemetry +prelude.emit_tool_call( + tool_name="read_file", + tool_args="/etc/shadow", + ok=True, + error=None +) +``` + +The resulting OTLP log record contains: +- `gen_ai_tool_name`: Name of the invoked tool (e.g. `read_file`, `run_shell`). +- `gen_ai_tool_args`: Formatted arguments passed to the tool. +- `ai_session_id`: Unique identifier correlating multi-step tool calls. +- `otel_status_code`: `OK` or `ERROR`. + +### Target Alert Patterns for AI Agents + +1. **Sensitive File Probing (`AIAgent-Sensitive-File-Access`)**: + ```sql + SELECT * FROM "agent_logs" + WHERE ( + gen_ai_tool_args LIKE '%/etc/shadow%' OR + gen_ai_tool_args LIKE '%/etc/sudoers%' OR + gen_ai_tool_args LIKE '%/etc/ssh/sshd_config%' OR + gen_ai_tool_args LIKE '%authorized_keys%' OR + gen_ai_tool_args LIKE '%/etc/ld.so.preload%' + ) + ``` +2. **Tool Failure Spike (`AIAgent-Tool-Error-Spike`)**: + Detects prompt-injection scanning or broken integrations: + ```sql + SELECT count(*) as error_count FROM "agent_logs" WHERE otel_status_code = 'ERROR' + ``` + Trigger condition: $\ge 5$ errors within a 5-minute sliding window. + +--- + +## 8. The Innocent Trigger Testing Workflow + +Every new detection alert must be accompanied by an **innocent validation trigger** — a safe, non-destructive command that exercises the exact detection rule without altering system state, escalating privileges, or running malware. + +### Step-by-Step Validation Lifecycle + +``` +┌─────────────────────────────────┐ +│ 1. Threat Modeling & MITRE Map │ -> Identify attack technique and required log stream +└────────────────┬────────────────┘ + ▼ +┌─────────────────────────────────┐ +│ 2. Author Detection Rule │ -> Create Falco / Sigma / OSquery / OpenObserve rule +└────────────────┬────────────────┘ + ▼ +┌─────────────────────────────────┐ +│ 3. Craft Innocent Trigger │ -> Formulate benign, non-destructive test command +└────────────────┬────────────────┘ + ▼ +┌─────────────────────────────────┐ +│ 4. Execute & Verify Pipelines │ -> Run via tools/trigger_alerts.py and check hits +└────────────────┬────────────────┘ + ▼ +┌─────────────────────────────────┐ +│ 5. Curate into Skill & Tests │ -> Update security-alert-triggers skill & pytest +└─────────────────────────────────┘ +``` + +### Adding New Triggers to `tools/trigger_alerts.py` + +When authoring a new trigger, add the function to [`tools/trigger_alerts.py`](../tools/trigger_alerts.py): + +```python +def trigger_custom_ioc() -> bool: + """Safe, non-destructive validation for new IoC detection.""" + print("[+] Triggering Custom IoC Alert...") + # Safe, read-only or harmless execution: + try: + subprocess.run(["stat", "/etc/hosts"], check=True) + return True + except Exception as e: + print(f"[-] Trigger failed: {e}") + return False +``` + +Then verify execution across all streams: + +```bash +python3 tools/trigger_alerts.py --all +``` + +--- + +## 9. Tuning, Suppression & False-Positive Management + +To maintain high signal-to-noise ratios, apply tuning at the earliest possible stage in the processing pipeline: + +1. **Kernel Stage (Falco Exceptions)**: + Add authorized process exceptions directly to rule conditions: + ```yaml + and not proc.name in (my_authorized_backup_daemon, logrotate) + ``` +2. **Collector Stage (OTTL Processors)**: + Drop unneeded high-volume telemetry before network export using OTTL conditions in `otel-collector-config.yaml`. +3. **Analytics Stage (OpenObserve SQL)**: + Filter expected maintenance commands or known user IDs: + ```sql + WHERE message LIKE '%sudo:%' AND message NOT LIKE '%authorized_admin%' + ``` +4. **Alert Receiver Stage (Action Playbooks)**: + Configure silence periods (`"silence": 30`) in `alerts.json` to prevent notification storms when an alert fires repeatedly within a short window. + +--- + +## 10. Related References & Runbooks + +- [Alert Payload Schema & Action Playbook Specification](./alerting_payload_schema.md) +- [Security Alert Triggers Agent Skill](file:///home/john/LocalObserve/.agents/skills/security-alert-triggers/SKILL.md) +- [OpenObserve VRL & Log Enrichment Guide](./openobserve_vrl_enrichment.md) +- [Multi-Runtime & Alerting Deployment Guide](./runtimes_alerting_and_resource_guide.md) +- [MITRE ATT&CK Linux Coverage Gaps & Mapping](./mitre_linux_coverage_gaps.md) +- [Detection Rule Review & Audit](./detection_rule_review.md) diff --git a/tools/trigger_alerts.py b/tools/trigger_alerts.py new file mode 100755 index 0000000..7f9bfa2 --- /dev/null +++ b/tools/trigger_alerts.py @@ -0,0 +1,476 @@ +#!/usr/bin/env python3 +"""trigger_alerts.py - Safe, innocent security alert trigger runner for LocalObserve. + +Triggers detection alerts across Falco, OSquery, ClamAV, RSigma, and AI Agent +observability using benign, non-destructive commands. Validates stream ingestion +in OpenObserve and webhook dispatch in alert-receiver, and optionally triggers +native Linux desktop notifications. + +Usage: + python3 tools/trigger_alerts.py --all + python3 tools/trigger_alerts.py --all --notify + python3 tools/trigger_alerts.py --stream falco + python3 tools/trigger_alerts.py --stream osquery + python3 tools/trigger_alerts.py --stream clamav + python3 tools/trigger_alerts.py --stream agent + python3 tools/trigger_alerts.py --stream rsigma + python3 tools/trigger_alerts.py --stream system-logs + python3 tools/trigger_alerts.py --verify + python3 tools/trigger_alerts.py --notify +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# Default endpoints and credentials +OO_URL = os.environ.get("OPENOBSERVE_URL", "http://localhost:5080") +OO_USER = os.environ.get("ZO_ROOT_USER_EMAIL", "root@example.com") +OO_PASS = os.environ.get("ZO_ROOT_USER_PASSWORD", "Complexpass#123") +RSIGMA_URL = os.environ.get("RSIGMA_URL", "http://localhost:9090") +ALERT_RECEIVER_URL = os.environ.get("ALERT_RECEIVER_URL", "http://localhost:9000") + + +def _oo_auth_header() -> dict[str, str]: + auth = base64.b64encode(f"{OO_USER}:{OO_PASS}".encode()).decode() + return {"Authorization": f"Basic {auth}", "Content-Type": "application/json"} + + +def _query_oo(stream: str, query_sql: str | None = None, minutes: int = 15, limit: int = 5) -> list[dict]: + """Execute SQL query against OpenObserve search endpoint.""" + now_us = int(time.time()) * 1_000_000 + sql = query_sql or f'SELECT * FROM "{stream}" ORDER BY _timestamp DESC LIMIT {limit}' + payload = { + "query": { + "sql": sql, + "start_time": now_us - minutes * 60 * 1_000_000, + "end_time": now_us, + } + } + req = urllib.request.Request( + f"{OO_URL}/api/default/_search", + data=json.dumps(payload).encode(), + headers=_oo_auth_header(), + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as r: + res = json.loads(r.read()) + return res.get("hits", []) + except Exception as e: + print(f"[-] OpenObserve search failed for stream '{stream}': {e}", file=sys.stderr) + return [] + + +def _post_json(url: str, data: dict | list, headers: dict[str, str] | None = None) -> tuple[int, str]: + hdrs = {"Content-Type": "application/json"} + if headers: + hdrs.update(headers) + req = urllib.request.Request( + url, + data=json.dumps(data).encode(), + headers=hdrs, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status, r.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.read().decode() if e.fp else str(e) + except Exception as e: + return 0, str(e) + + +# --------------------------------------------------------------------------- +# Individual Stream Triggers (All Non-Destructive & Innocent) +# --------------------------------------------------------------------------- + + +def trigger_falco() -> bool: + """Trigger Falco detections using safe, innocent commands.""" + print("\n--- [1/6] Triggering Falco Alerts ---") + success = True + + # 1. Read sensitive file (safe read of pwquality.conf) -> Read sensitive file untrusted / Falco-High-Severity + try: + print("[+] 1.1 Read Sensitive File (Innocent read of /etc/security/pwquality.conf)") + with open("/etc/security/pwquality.conf", "r") as f: + _ = f.read(10) + print(" -> Trigger executed successfully") + except Exception as e: + print(f" [-] Trigger failed: {e}") + success = False + + # 2. Execution from /tmp (benign /bin/true copied and executed) -> Execution from Temporary Directory + try: + print("[+] 1.2 Temp Directory Execution (Copying and running /bin/true in /tmp)") + tmp_bin = Path("/tmp/innocent_alert_test_bin") + subprocess.run(["cp", "/bin/true", str(tmp_bin)], check=True) + subprocess.run([str(tmp_bin)], check=True) + tmp_bin.unlink(missing_ok=True) + print(" -> Trigger executed and cleaned up successfully") + except Exception as e: + print(f" [-] Trigger failed: {e}") + success = False + + # 3. LD_PRELOAD injection (loading standard benign libc) -> Hijack Execution Flow with LD_PRELOAD + try: + print("[+] 1.3 LD_PRELOAD Environment Hook (Preloading standard libc.so.6 on benign true)") + libc_path = "/lib/x86_64-linux-gnu/libc.so.6" + if os.path.exists(libc_path): + env = os.environ.copy() + env["LD_PRELOAD"] = libc_path + subprocess.run(["true"], env=env, check=True) + print(" -> Trigger executed successfully") + else: + print(f" [*] Notice: {libc_path} not found on this arch, skipping LD_PRELOAD") + except Exception as e: + print(f" [-] Trigger failed: {e}") + success = False + + return success + + +def trigger_osquery() -> bool: + """Trigger OSquery alerts by triggering FIM and recording monitored events.""" + print("\n--- [2/6] Triggering OSquery Alerts ---") + success = True + + # 1. FIM file modification on monitored bashrc + try: + print("[+] 2.1 FIM File Update (Touching user ~/.bashrc)") + bashrc = Path.home() / ".bashrc" + if bashrc.exists(): + bashrc.touch() + print(" -> Touched ~/.bashrc successfully") + except Exception as e: + print(f" [-] Failed touching ~/.bashrc: {e}") + + # 2. FIM touch inside osquery container if running + try: + print("[+] 2.2 FIM Container File Touch (/root/.bashrc inside container)") + res = subprocess.run( + ["docker", "exec", "localobserve-osquery-1", "touch", "/root/.bashrc"], + capture_output=True, + text=True, + ) + if res.returncode == 0: + print(" -> Container FIM trigger executed successfully") + else: + print(f" [*] Notice: container touch returned {res.returncode}: {res.stderr.strip()}") + except Exception as e: + print(f" [*] Notice: docker exec skipped ({e})") + + # 3. Synthetic detection events in osquery results log for instant pipeline validation + try: + print("[+] 2.3 OSquery Telemetry Results Event (Crontab & SUID changes)") + log_file = REPO_ROOT / ".data" / "osquery" / "osqueryd.results.log" + if not log_file.exists(): + log_file.parent.mkdir(parents=True, exist_ok=True) + log_file.touch() + + # Fix permissions if container root created it + try: + subprocess.run( + ["docker", "exec", "localobserve-osquery-1", "chmod", "666", "/var/log/osquery/osqueryd.results.log"], + capture_output=True, + ) + except Exception: + pass + + now_str = time.asctime(time.gmtime()) + now_ts = int(time.time()) + + events = [ + { + "name": "crontab", + "command": "curl -s http://localhost:5080/healthz", + "columns": { + "command": "curl -s http://localhost:5080/healthz", + "path": "/var/spool/cron/crontabs/testuser", + }, + "action": "added", + "hostIdentifier": "localhost", + "calendarTime": now_str, + "unixTime": now_ts, + }, + { + "name": "file_events", + "action": "UPDATED", + "columns": { + "target_path": "/home/john/.bashrc", + "action": "UPDATED", + }, + "hostIdentifier": "localhost", + "calendarTime": now_str, + "unixTime": now_ts, + }, + { + "name": "suid_bin_changes", + "action": "added", + "columns": { + "path": "/tmp/innocent_suid_test", + "permissions": "4755", + }, + "hostIdentifier": "localhost", + "calendarTime": now_str, + "unixTime": now_ts, + }, + ] + with open(log_file, "a") as f: + for ev in events: + f.write(json.dumps(ev) + "\n") + print(" -> Appended crontab, file_events, and suid_bin_changes telemetry to osquery log") + except Exception as e: + print(f" [-] Appending osquery events failed: {e}") + success = False + + return success + + +def trigger_clamav() -> bool: + """Trigger ClamAV alert by logging benign EICAR test scan result.""" + print("\n--- [3/6] Triggering ClamAV Alerts ---") + try: + clamav_log = REPO_ROOT / ".data" / "clamav" / "scan.log" + clamav_log.parent.mkdir(parents=True, exist_ok=True) + # EICAR detection line conforming to collector regex + line = "/tmp/innocent_eicar_test.com: FOUND Win.Test.EICAR_HDB-1\n" + with open(clamav_log, "a") as f: + f.write(line) + print("[+] Logged harmless EICAR test detection to .data/clamav/scan.log") + return True + except Exception as e: + print(f"[-] ClamAV trigger failed: {e}") + return False + + +def trigger_agent() -> bool: + """Trigger AI Agent observability alerts via synthetic tool calls.""" + print("\n--- [4/6] Triggering AI Agent Observability Alerts ---") + try: + demo_script = REPO_ROOT / "tools" / "agent_session_demo.py" + venv_py = REPO_ROOT / ".venv" / "bin" / "python" + uv_bin = shutil.which("uv") or str(Path.home() / ".local" / "bin" / "uv") + + if venv_py.exists(): + cmd = [str(venv_py), str(demo_script), "--quick"] + elif os.path.exists(uv_bin): + cmd = [uv_bin, "run", "python", str(demo_script), "--quick"] + else: + cmd = [sys.executable, str(demo_script), "--quick"] + + res = subprocess.run( + cmd, + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + timeout=30, + ) + print(res.stdout.strip()) + if res.returncode == 0: + print("[+] Emitted AI agent tool telemetry (sensitive file read + error call)") + return True + else: + print(f"[-] AI agent demo exited with code {res.returncode}: {res.stderr}") + return False + except Exception as e: + print(f"[-] AI Agent trigger failed: {e}") + return False + + +def trigger_rsigma() -> bool: + """Trigger RSigma streaming Sigma detections.""" + print("\n--- [5/6] Triggering RSigma Real-Time Detections ---") + success = True + test_events = [ + { + "name": "unshare", + "cmdline": "unshare --user --map-root-user /bin/bash", + "pid": 88000 + int(time.time()) % 1000, + }, + { + "service_namespace": "agent", + "gen_ai_tool_name": "read_file", + "gen_ai_tool_args": "/etc/shadow", + "pid": 89000 + int(time.time()) % 1000, + }, + { + "service_namespace": "agent", + "gen_ai_tool_name": "run_shell", + "gen_ai_tool_args": "rm -rf /", + "pid": 90000 + int(time.time()) % 1000, + }, + ] + + for ev in test_events: + status, resp = _post_json(f"{RSIGMA_URL}/api/v1/events", ev) + rule_hint = ev.get("name") or ev.get("gen_ai_tool_name") + if status == 200: + print(f"[+] RSigma event accepted ({rule_hint}): {resp.strip()}") + else: + print(f"[-] RSigma event rejected ({rule_hint}): HTTP {status} {resp.strip()}") + success = False + + return success + + +def trigger_system_logs() -> bool: + """Trigger SystemLogs auth failure and sudo alerts via OpenObserve ingest.""" + print("\n--- [6/6] Triggering SystemLogs Alerts ---") + logs = [ + { + "message": "sshd[99123]: Failed password for invalid user admin from 192.168.1.100 port 55123 ssh2", + "facility": "auth", + "hostname": "localobserve-test", + }, + { + "message": "sudo: guest_user : TTY=pts/3 ; PWD=/tmp ; USER=root ; COMMAND=/bin/cat /etc/shadow", + "facility": "authpriv", + "hostname": "localobserve-test", + }, + ] + status, resp = _post_json( + f"{OO_URL}/api/default/system_logs/_json", + logs, + headers=_oo_auth_header(), + ) + if status == 200: + print(f"[+] System logs ingested successfully into system_logs stream: {resp.strip()}") + return True + else: + print(f"[-] System logs ingestion failed: HTTP {status} {resp.strip()}") + return False + + +# --------------------------------------------------------------------------- +# Verification Engine & Desktop Notification Replay +# --------------------------------------------------------------------------- + + +def replay_desktop_notifications() -> None: + """Invoke host-side desktop notifier to replay queued alert popups via notify-send.""" + print("\n--- Triggering Native Desktop Notifications (notify-send) ---") + notifier_script = REPO_ROOT / "tools" / "desktop_notifier.py" + if notifier_script.exists(): + try: + res = subprocess.run( + [sys.executable, str(notifier_script), "--replay", "--once"], + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + timeout=15, + ) + print(res.stdout.strip()) + if res.returncode == 0: + print("[✔] Desktop notifications successfully dispatched via notify-send") + else: + print(f"[!] Desktop notifier exited with code {res.returncode}: {res.stderr.strip()}") + except Exception as e: + print(f"[-] Failed executing desktop notifier: {e}") + else: + print("[-] tools/desktop_notifier.py not found") + + +def verify_all(minutes: int = 15) -> dict[str, int]: + """Verify detection events across all OpenObserve streams and webhook receiver.""" + print("\n=======================================================") + print(f"Verifying Ingested Telemetry Across Streams (Last {minutes}m)") + print("=======================================================") + + results = {} + streams = { + "falco": 'SELECT count(*) as cnt FROM "falco"', + "clamav": 'SELECT count(*) as cnt FROM "clamav"', + "agent_logs": 'SELECT count(*) as cnt FROM "agent_logs"', + "system_logs": 'SELECT count(*) as cnt FROM "system_logs"', + "osquery": 'SELECT count(*) as cnt FROM "osquery"', + } + + for s, sql in streams.items(): + hits = _query_oo(s, query_sql=sql, minutes=minutes) + cnt = hits[0].get("cnt", 0) if hits else 0 + results[s] = cnt + status_icon = "✔" if cnt > 0 else "✘" + print(f" [{status_icon}] Stream '{s:<12}': {cnt:>6} hits") + + # Check alert-receiver alerts directory + print("\n--- Verifying Alert Receiver Dispatches ---") + try: + proc = subprocess.run( + ["docker", "exec", "localobserve-alert-receiver-1", "ls", "-1", "/var/log/alerts"], + capture_output=True, + text=True, + timeout=5, + ) + alert_files = [f for f in proc.stdout.splitlines() if f.endswith(".json")] + print(f" [✔] Alert-Receiver dispatched alerts: {len(alert_files)} files in /var/log/alerts") + if alert_files: + latest = alert_files[-1] + cat_proc = subprocess.run( + ["docker", "exec", "localobserve-alert-receiver-1", "cat", f"/var/log/alerts/{latest}"], + capture_output=True, + text=True, + timeout=5, + ) + print(f" Latest alert ({latest}):\n{cat_proc.stdout.strip()}") + except Exception as e: + print(f" [-] Alert-Receiver check skipped or failed: {e}") + + return results + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--all", action="store_true", help="Trigger all innocent security alerts") + parser.add_argument("--stream", choices=["falco", "osquery", "clamav", "agent", "rsigma", "system-logs"], help="Trigger specific stream") + parser.add_argument("--verify", action="store_true", help="Verify ingestion counts in OpenObserve") + parser.add_argument("--notify", action="store_true", help="Replay alert backlog to native Linux desktop notifications via notify-send") + parser.add_argument("--window", type=int, default=15, help="Search time window in minutes for verification") + args = parser.parse_args() + + if not (args.all or args.stream or args.verify or args.notify): + parser.print_help() + return 1 + + if args.all or args.stream == "falco": + trigger_falco() + if args.all or args.stream == "osquery": + trigger_osquery() + if args.all or args.stream == "clamav": + trigger_clamav() + if args.all or args.stream == "agent": + trigger_agent() + if args.all or args.stream == "rsigma": + trigger_rsigma() + if args.all or args.stream == "system-logs": + trigger_system_logs() + + # Allow pipelines 3s to process and ship logs + if args.all or args.stream: + print("\nWaiting 3 seconds for OpenTelemetry Collector and OpenObserve to process...") + time.sleep(3) + + if args.all or args.verify: + verify_all(minutes=args.window) + + if args.all or args.notify: + replay_desktop_notifications() + + return 0 + + +if __name__ == "__main__": + sys.exit(main())