diff --git a/deploy/incident-broker/broker.py b/deploy/incident-broker/broker.py index 06f05f1..e02e018 100644 --- a/deploy/incident-broker/broker.py +++ b/deploy/incident-broker/broker.py @@ -480,7 +480,7 @@ def remediation_snapshot(db, root): presentations = {} if not isinstance(presentations, dict): presentations = {} cases = [] - pending_releases = set() + retained_issues = set() scheduler_row = db.execute("select value from meta where key='scheduler'").fetchone() scheduler = json.loads(scheduler_row['value']) if scheduler_row else {} waits = {r['eventId']: r for r in scheduler.get('queued', [])} @@ -491,9 +491,9 @@ def bounded(value, limit=800): for ident, rows in groups.items(): latest = max(rows, key=lambda r: int(r['id'].split(':')[1])) issue = json.loads(latest['body'])['issue'] - if issue['domain'] == 'maintenance' and issue['status'] == 'open': - pending_releases.add(ident) active = [r for r in rows if r['status'] in {'running', 'uncertain'}] + if active or any(r['status'] == 'queued' for r in rows) or (issue['domain'] == 'maintenance' and issue['status'] == 'open'): + retained_issues.add(ident) current = max(active, key=lambda r: r['started'] or 0) if active else latest finished = [r for r in rows if r['status'] == 'completed' and r['result']] correlation = db.execute('select * from event_correlations where event_id=?', (current['id'],)).fetchone() @@ -527,11 +527,11 @@ def bounded(value, limit=800): 'nextAction': bounded(detail.get('nextAction')), 'changes': [bounded(x, 300) for x in detail.get('changes', [])[:8] if isinstance(x, str)] if isinstance(detail.get('changes'), list) else []} if receipt else None}) - # Release tasks emit no reminders. Retain their completion channel until - # the producer sends the resolved revision acknowledging the receipt. + # Never trim current ownership or queued work out of the board. Release + # tasks also retain their completion channel until producer acknowledgement. recent = cases[-1000:] shown = {c['issueId'] for c in recent} - retained = [c for c in cases if c['issueId'] in pending_releases and c['issueId'] not in shown] + recent + retained = [c for c in cases if c['issueId'] in retained_issues and c['issueId'] not in shown] + recent return {'schemaVersion': 1, 'generatedAt': stamp(time.time()), 'enabled': (root / 'ENABLED').exists() and dispatch_on(db), 'maxActive': scheduler.get('maxActive'), 'progress': progress_health(db), 'cases': retained, 'truncated': len(cases) > len(retained)} diff --git a/deploy/incident-broker/test_broker.py b/deploy/incident-broker/test_broker.py index d83d60a..017ff79 100644 --- a/deploy/incident-broker/test_broker.py +++ b/deploy/incident-broker/test_broker.py @@ -632,6 +632,26 @@ def test_release_completion_survives_board_trimming_until_producer_acknowledges( self.assertEqual(len(snapshot['cases']), 1000) self.assertNotIn('b'*24, [c['issueId'] for c in snapshot['cases']]) + def test_board_retains_old_queued_and_held_issues_beyond_history_limit(self): + root = pathlib.Path(self.temp.name) + queued = event(); held = event() + held['eventId'] = 'b'*24+':1'; held['issue']['id'] = 'b'*24 + b.ingest(self.db, envelope(queued, held)) + self.db.execute("update events set status='uncertain',started=?,finished=?,result=? where id=?", (time.time()-1800, time.time()-1200, '1'*24, held['eventId'])) + self.db.commit() + for start in range(0, 1000, 50): + batch = [] + for n in range(start, start + 50): + e = event(transition='resolved'); ident = hashlib.sha256(str(n).encode()).hexdigest()[:24] + e['eventId'] = ident+':1'; e['issue']['id'] = ident; batch.append(e) + b.ingest(self.db, envelope(*batch)) + snapshot = b.remediation_snapshot(self.db, root) + cases = {c['issueId']: c for c in snapshot['cases']} + self.assertEqual(len(cases), 1002) + self.assertEqual(cases['a'*24]['pendingEvents'], 1) + self.assertTrue(cases['b'*24]['active']) + self.assertTrue(cases['b'*24]['attentionRequired']) + def test_an_attempt_that_did_no_work_returns_to_the_queue_after_its_lease(self): root,run,now,failed=self.unstarted_run() def broken(_): raise OSError('gateway unavailable') diff --git a/server/remediation-activity.js b/server/remediation-activity.js new file mode 100644 index 0000000..2dfee5b --- /dev/null +++ b/server/remediation-activity.js @@ -0,0 +1,25 @@ +// Display history only. Never changes monitoring, delivery, receipts or locks. +import { join } from 'node:path'; +import { readJSON, writeAtomic } from '../shared/settings.js'; + +const path = (dataDir) => join(dataDir, 'remediation-activity.json'); +export function loadActivity(dataDir) { + const value = readJSON(path(dataDir)); + if (value?.schemaVersion !== 1 || !Number.isFinite(Date.parse(value.clearedAt)) || !value.revisions || typeof value.revisions !== 'object' || Array.isArray(value.revisions) + || Object.values(value.revisions).some((n) => !Number.isSafeInteger(n) || n < 0)) return null; + return value; +} + +export function clearActivity(dataDir, state, actor, now = Date.now()) { + const revisions = Object.fromEntries((state?.issues || []).map((i) => [i.id, Number.isSafeInteger(i.revision) ? i.revision : 0])); + const value = { schemaVersion: 1, clearedAt: new Date(now).toISOString(), clearedBy: String(actor || '').slice(0, 39), revisions }; + writeAtomic(path(dataDir), JSON.stringify(value)); + return { clearedAt: value.clearedAt, clearedBy: value.clearedBy }; +} + +export function activityHidden(issue, response, state) { + // Always retain current ownership and queued work, including stale feeds. + if (response?.active || response?.workerState === 'queued' || response?.pendingEvents > 0 || state?.outbox?.some((e) => e.issue?.id === issue.id)) return false; + const revisions = state?.activityHistory?.revisions; + return !!revisions && Object.hasOwn(revisions, issue.id) && (issue.revision || 0) <= revisions[issue.id]; +} diff --git a/server/remediation-activity.test.js b/server/remediation-activity.test.js new file mode 100644 index 0000000..a1fe8d2 --- /dev/null +++ b/server/remediation-activity.test.js @@ -0,0 +1,38 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { clearActivity, loadActivity } from './remediation-activity.js'; +import { remediationView } from './remediation.js'; + +test('history clear survives reload, retains queued/held work, and new revisions reappear without changing monitoring', (t) => { + const root = mkdtempSync(join(tmpdir(), 'activity-')); t.after(() => rmSync(root, { recursive: true, force: true })); + const now = Date.now(), at = new Date(now).toISOString(); + const issues = ['old', 'queued', 'held'].map((id) => ({ id, revision: 2, domain: 'ci', severity: 'warning', status: 'open' })); + const state = { generatedAt: at, issues, outbox: [], delivery: { receiver: { workerAt: now / 1000, counts: { queued: 3 }, remediation: { + schemaVersion: 1, generatedAt: at, enabled: false, truncated: true, progress: { attentionRequired: true, stalledRuns: 1 }, cases: [ + { issueId: 'queued', workerState: 'queued', pendingEvents: 3 }, { issueId: 'held', active: true, workerState: 'uncertain', attentionRequired: true }, + ], + } } } }; + const original = structuredClone(state); + clearActivity(root, state, 'PRIVATE-ADMIN', now); + assert.deepEqual(state, original, 'display reset changes no issue, event or ownership'); + state.activityHistory = loadActivity(root); + const view = remediationView(state, { full: true, now }); + assert.deepEqual(view.cases.map((c) => c.id).sort(), ['held', 'queued']); + assert.equal(view.activityHistory.hidden, 1); assert.equal(view.truncated, false); + assert.equal(view.execution, 'stalled'); assert.equal(view.stalledRuns, 1); assert.equal(view.queuedEvents, 3); + assert.doesNotMatch(JSON.stringify(remediationView(state, { now })), /PRIVATE/); + issues[0].revision++; + assert.equal(remediationView(state, { full: true, now }).cases.length, 3); + assert.equal(remediationView(state, { full: true, now }).truncated, true, 'missing new history stays visible'); + writeFileSync(join(root, 'remediation-activity.json'), '{broken'); + assert.equal(loadActivity(root), null, 'invalid reset must not hide anything'); +}); + +test('administrator queue total uses receiver totals even when the case feed is truncated', () => { + const state = { issues: [], delivery: { receiver: { counts: { queued: 7 }, remediation: { cases: [], truncated: true } } } }; + assert.equal(remediationView(state, { full: true }).queuedEvents, 7); + assert.equal(remediationView(state).queuedEvents, 0, 'public network filtering cannot leak private queue counts'); +}); diff --git a/server/remediation.js b/server/remediation.js index 42fdbab..dfbf0f1 100644 --- a/server/remediation.js +++ b/server/remediation.js @@ -2,6 +2,7 @@ import { TITLES } from './incidents.js'; import { RELEASE_CODES } from './release-webhook.js'; import { stallReasons } from './stall-alert.js'; +import { activityHidden } from './remediation-activity.js'; const fresh = (at, now) => Number.isFinite(Date.parse(at)) && Date.parse(at) <= now + 60_000 && now - Date.parse(at) <= 180_000; const text = (value, size = 800) => typeof value === 'string' ? value.slice(0, size) : null; @@ -31,7 +32,9 @@ export function remediationView(state, { full = false, visibleNetworks = [], now const workerFresh = feed?.schemaVersion === 1 && fresh(feed.generatedAt, now) && fresh(workerAt, now) && Array.isArray(feed.cases); const observations = new Map((Array.isArray(feed?.cases) ? feed.cases : []).map((c) => [c.issueId, c])); const visible = new Set(visibleNetworks); - const cases = (state?.issues || []).filter((i) => full || i.domain !== 'network' || visible.has(i.scope)).map((issue) => { + const visibleIssues = (state?.issues || []).filter((i) => full || i.domain !== 'network' || visible.has(i.scope)); + const displayedIssues = visibleIssues.filter((i) => !activityHidden(i, observations.get(i.id), state)); + const cases = displayedIssues.map((issue) => { const response = observations.get(issue.id); const last = response?.lastResponse; let stage = 'unknown', reason = 'Waiting for remediation status'; @@ -96,12 +99,16 @@ export function remediationView(state, { full = false, visibleNetworks = [], now ...(full ? { reset: requested?.recovered === true, changedAt: iso(requested?.updatedAt), changedBy: text(requested?.updatedBy, 39), clearedAt: iso(requested?.clearedAt), clearedBy: text(requested?.clearedBy, 39) } : {}) }; return { schemaVersion: 1, public: !full, generatedAt: iso(state?.generatedAt), workerAt, control, stalled: attention.length > 0, attention: full ? attention : undefined, stalledSince: full ? iso(state?.stall?.since) : undefined, - stale: !producerFresh || !workerFresh, producerFresh, workerFresh, truncated: feed?.truncated === true, + stale: !producerFresh || !workerFresh, producerFresh, workerFresh, + truncated: feed?.truncated === true && displayedIssues.some((i) => !observations.has(i.id)), + activityHistory: { clearedAt: iso(state?.activityHistory?.clearedAt), hidden: visibleIssues.length - displayedIssues.length, + ...(full ? { clearedBy: text(state?.activityHistory?.clearedBy, 39) } : {}) }, dispatch: !workerFresh ? 'unknown' : feed.enabled ? 'enabled' : 'paused', execution: !workerFresh || typeof feed.progress?.attentionRequired !== 'boolean' ? 'unknown' : feed.progress.attentionRequired ? 'stalled' : 'healthy', stalledRuns: workerFresh && Number.isSafeInteger(feed.progress?.stalledRuns) ? feed.progress.stalledRuns : null, maxActive: Number.isSafeInteger(feed?.maxActive) ? feed.maxActive : null, delivery: { pending: full ? state?.outbox?.length || 0 : null, failed: !!state?.delivery?.error, lastAckAt: iso(state?.delivery?.lastAckAt) }, maintenance: Object.entries(state?.maintenance || {}).filter(([name]) => full || visible.has(name)).map(([network, value]) => ({ network, operations:full ? value.operations : undefined })), - counts, queuedEvents: cases.reduce((sum, c) => sum + c.pendingEvents, 0), cases }; + counts, queuedEvents: full && Number.isSafeInteger(state?.delivery?.receiver?.counts?.queued) + ? state.delivery.receiver.counts.queued : cases.reduce((sum, c) => sum + c.pendingEvents, 0), cases }; } diff --git a/server/web.js b/server/web.js index b34f9ac..ff92a7e 100644 --- a/server/web.js +++ b/server/web.js @@ -15,6 +15,7 @@ import { loadIncidentState, publicIssues, apiAuthorized, readSecret } from './in import { remediationView } from './remediation.js'; import { receiveRelease } from './release-webhook.js'; import { againstReceiver, loadControl, requestClear, setDispatch } from './remediation-control.js'; +import { clearActivity, loadActivity } from './remediation-activity.js'; import { publicInventory } from './aws-public.js'; import { COMPONENTS, COMPONENT_REPOS, accessFor, adminFor, loadSettings, memberOf, operatorFor, readJSON, saveSettings, validateSettings, writeAtomic } from '../shared/settings.js'; import { expectations } from '../shared/monitoring.js'; @@ -142,7 +143,7 @@ export function createWeb({ dataDir, origin, auth: authDeps, fetcher = fetch, cl reloadSettings(); const full = isAdmin(req) || apiAuthorized(req.get('authorization'), readSecret(process.env.INCIDENT_API_TOKEN_FILE)); const incidents = loadIncidentState(dataDir); - res.json(remediationView({ ...incidents, maintenance:operationMaintenance(dataDir, settings, clock()), control: againstReceiver(loadControl(dataDir), incidents?.delivery?.receiver?.control) }, { full, visibleNetworks: visible(req).map((n) => n.name), now: clock() })); + res.json(remediationView({ ...incidents, activityHistory: loadActivity(dataDir), maintenance:operationMaintenance(dataDir, settings, clock()), control: againstReceiver(loadControl(dataDir), incidents?.delivery?.receiver?.control) }, { full, visibleNetworks: visible(req).map((n) => n.name), now: clock() })); }); app.get('/api/me', (req, res) => { const u = user(req); @@ -388,6 +389,11 @@ export function createWeb({ dataDir, origin, auth: authDeps, fetcher = fetch, cl controlled(res, setDispatch(dataDir, req.body.enabled, req.session.user.login, clock(), acknowledged())); }); app.post('/api/remediation/clear', ...adminWrite, (req, res) => controlled(res, requestClear(dataDir, req.session.user.login, clock(), acknowledged()))); + app.post('/api/remediation/activity/clear', ...adminWrite, (req, res) => { + const activityHistory = clearActivity(dataDir, loadIncidentState(dataDir), req.session.user.login, clock()); + push('remediation', { at: new Date(clock()).toISOString() }); + res.json({ activityHistory }); + }); // Infrastructure pages: full detail for any account with access; before // sign-in a sanitized view (statistics and spend, nothing identifying). diff --git a/server/web.test.js b/server/web.test.js index df0ded8..f8b998c 100644 --- a/server/web.test.js +++ b/server/web.test.js @@ -317,9 +317,11 @@ test('remediation switch and queue clear are administrator-only, CSRF-protected try { assert.equal((await post(w, '/api/remediation/dispatch', { enabled: true })).status, 401); assert.equal((await post(w, '/api/remediation/clear')).status, 401); + assert.equal((await post(w, '/api/remediation/activity/clear')).status, 401); const csrf = await w.login(); assert.equal((await (await w.req('/api/remediation')).json()).control.requested, 'off'); assert.equal((await post(w, '/api/remediation/dispatch', { enabled: true })).status, 403, 'a session without the CSRF token is refused'); + assert.equal((await post(w, '/api/remediation/activity/clear')).status, 403); assert.equal((await post(w, '/api/remediation/dispatch', { enabled: 'yes' }, csrf)).status, 400); assert.equal((await post(w, '/api/remediation/dispatch', { enabled: true }, csrf)).status, 200); assert.deepEqual([stored(w).seq, stored(w).dispatch, stored(w).updatedBy], [1, 'on', 'someone']); @@ -342,12 +344,19 @@ test('remediation switch and queue clear are administrator-only, CSRF-protected assert.equal((await post(w, '/api/remediation/clear', undefined, csrf)).status, 200); assert.deepEqual([stored(w).seq, stored(w).dispatch, stored(w).clearSeq], [11, 'off', 11], 'a clear after the loss is not stale either, and does not switch on'); assert.equal((await (await w.req('/api/remediation')).json()).control.reset, false); + const beforeHistoryClear = stored(w); + const beforeIssues = readFileSync(join(w.dataDir, 'incidents', 'state.json'), 'utf8'); + assert.equal((await post(w, '/api/remediation/activity/clear', undefined, csrf)).status, 200); + assert.deepEqual(stored(w), beforeHistoryClear, 'activity clear never changes dispatch or the queue'); + assert.equal(readFileSync(join(w.dataDir, 'incidents', 'state.json'), 'utf8'), beforeIssues); + assert.equal((await (await w.req('/api/remediation')).json()).activityHistory.clearedBy, 'someone'); } finally { w.close(); } const member = await start(42); try { const csrf = await member.login(); assert.equal((await post(member, '/api/remediation/dispatch', { enabled: true }, csrf)).status, 403); assert.equal((await post(member, '/api/remediation/clear', undefined, csrf)).status, 403); + assert.equal((await post(member, '/api/remediation/activity/clear', undefined, csrf)).status, 403); assert.equal((await (await member.req('/api/remediation')).json()).control.changedBy, undefined); } finally { member.close(); } }); diff --git a/src/pages/Remediation.jsx b/src/pages/Remediation.jsx index 4f34df8..5f9768e 100644 --- a/src/pages/Remediation.jsx +++ b/src/pages/Remediation.jsx @@ -55,6 +55,7 @@ export default function Remediation() {
Counts are unique issues. {data.queuedEvents} queued events may include repeat updates. A completed investigation is not a verified fix.
Activity cleared {ago(data.activityHistory.clearedAt, now)} ago · {data.activityHistory.hidden} older entries hidden. New events and queued or active repairs remain visible. Monitoring findings are unchanged.
}