Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions deploy/incident-broker/broker.py
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,7 @@ def remediation_snapshot(db, root):
presentations = {}
if not isinstance(presentations, dict): presentations = {}
cases = []
pending_releases = set()
retained_issues = set()
scheduler_row = db.execute("select value from meta where key='scheduler'").fetchone()
scheduler = json.loads(scheduler_row['value']) if scheduler_row else {}
waits = {r['eventId']: r for r in scheduler.get('queued', [])}
Expand All @@ -491,9 +491,9 @@ def bounded(value, limit=800):
for ident, rows in groups.items():
latest = max(rows, key=lambda r: int(r['id'].split(':')[1]))
issue = json.loads(latest['body'])['issue']
if issue['domain'] == 'maintenance' and issue['status'] == 'open':
pending_releases.add(ident)
active = [r for r in rows if r['status'] in {'running', 'uncertain'}]
if active or any(r['status'] == 'queued' for r in rows) or (issue['domain'] == 'maintenance' and issue['status'] == 'open'):
retained_issues.add(ident)
current = max(active, key=lambda r: r['started'] or 0) if active else latest
finished = [r for r in rows if r['status'] == 'completed' and r['result']]
correlation = db.execute('select * from event_correlations where event_id=?', (current['id'],)).fetchone()
Expand Down Expand Up @@ -527,11 +527,11 @@ def bounded(value, limit=800):
'nextAction': bounded(detail.get('nextAction')),
'changes': [bounded(x, 300) for x in detail.get('changes', [])[:8] if isinstance(x, str)] if isinstance(detail.get('changes'), list) else []}
if receipt else None})
# Release tasks emit no reminders. Retain their completion channel until
# the producer sends the resolved revision acknowledging the receipt.
# Never trim current ownership or queued work out of the board. Release
# tasks also retain their completion channel until producer acknowledgement.
recent = cases[-1000:]
shown = {c['issueId'] for c in recent}
retained = [c for c in cases if c['issueId'] in pending_releases and c['issueId'] not in shown] + recent
retained = [c for c in cases if c['issueId'] in retained_issues and c['issueId'] not in shown] + recent
return {'schemaVersion': 1, 'generatedAt': stamp(time.time()), 'enabled': (root / 'ENABLED').exists() and dispatch_on(db),
'maxActive': scheduler.get('maxActive'), 'progress': progress_health(db), 'cases': retained, 'truncated': len(cases) > len(retained)}

Expand Down
20 changes: 20 additions & 0 deletions deploy/incident-broker/test_broker.py
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,26 @@ def test_release_completion_survives_board_trimming_until_producer_acknowledges(
self.assertEqual(len(snapshot['cases']), 1000)
self.assertNotIn('b'*24, [c['issueId'] for c in snapshot['cases']])

def test_board_retains_old_queued_and_held_issues_beyond_history_limit(self):
root = pathlib.Path(self.temp.name)
queued = event(); held = event()
held['eventId'] = 'b'*24+':1'; held['issue']['id'] = 'b'*24
b.ingest(self.db, envelope(queued, held))
self.db.execute("update events set status='uncertain',started=?,finished=?,result=? where id=?", (time.time()-1800, time.time()-1200, '1'*24, held['eventId']))
self.db.commit()
for start in range(0, 1000, 50):
batch = []
for n in range(start, start + 50):
e = event(transition='resolved'); ident = hashlib.sha256(str(n).encode()).hexdigest()[:24]
e['eventId'] = ident+':1'; e['issue']['id'] = ident; batch.append(e)
b.ingest(self.db, envelope(*batch))
snapshot = b.remediation_snapshot(self.db, root)
cases = {c['issueId']: c for c in snapshot['cases']}
self.assertEqual(len(cases), 1002)
self.assertEqual(cases['a'*24]['pendingEvents'], 1)
self.assertTrue(cases['b'*24]['active'])
self.assertTrue(cases['b'*24]['attentionRequired'])

def test_an_attempt_that_did_no_work_returns_to_the_queue_after_its_lease(self):
root,run,now,failed=self.unstarted_run()
def broken(_): raise OSError('gateway unavailable')
Expand Down
25 changes: 25 additions & 0 deletions server/remediation-activity.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Display history only. Never changes monitoring, delivery, receipts or locks.
import { join } from 'node:path';
import { readJSON, writeAtomic } from '../shared/settings.js';

const path = (dataDir) => join(dataDir, 'remediation-activity.json');
export function loadActivity(dataDir) {
const value = readJSON(path(dataDir));
if (value?.schemaVersion !== 1 || !Number.isFinite(Date.parse(value.clearedAt)) || !value.revisions || typeof value.revisions !== 'object' || Array.isArray(value.revisions)
|| Object.values(value.revisions).some((n) => !Number.isSafeInteger(n) || n < 0)) return null;
return value;
}

export function clearActivity(dataDir, state, actor, now = Date.now()) {
const revisions = Object.fromEntries((state?.issues || []).map((i) => [i.id, Number.isSafeInteger(i.revision) ? i.revision : 0]));
const value = { schemaVersion: 1, clearedAt: new Date(now).toISOString(), clearedBy: String(actor || '').slice(0, 39), revisions };
writeAtomic(path(dataDir), JSON.stringify(value));
return { clearedAt: value.clearedAt, clearedBy: value.clearedBy };
}

export function activityHidden(issue, response, state) {
// Always retain current ownership and queued work, including stale feeds.
if (response?.active || response?.workerState === 'queued' || response?.pendingEvents > 0 || state?.outbox?.some((e) => e.issue?.id === issue.id)) return false;
const revisions = state?.activityHistory?.revisions;
return !!revisions && Object.hasOwn(revisions, issue.id) && (issue.revision || 0) <= revisions[issue.id];
}
38 changes: 38 additions & 0 deletions server/remediation-activity.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { clearActivity, loadActivity } from './remediation-activity.js';
import { remediationView } from './remediation.js';

test('history clear survives reload, retains queued/held work, and new revisions reappear without changing monitoring', (t) => {
const root = mkdtempSync(join(tmpdir(), 'activity-')); t.after(() => rmSync(root, { recursive: true, force: true }));
const now = Date.now(), at = new Date(now).toISOString();
const issues = ['old', 'queued', 'held'].map((id) => ({ id, revision: 2, domain: 'ci', severity: 'warning', status: 'open' }));
const state = { generatedAt: at, issues, outbox: [], delivery: { receiver: { workerAt: now / 1000, counts: { queued: 3 }, remediation: {
schemaVersion: 1, generatedAt: at, enabled: false, truncated: true, progress: { attentionRequired: true, stalledRuns: 1 }, cases: [
{ issueId: 'queued', workerState: 'queued', pendingEvents: 3 }, { issueId: 'held', active: true, workerState: 'uncertain', attentionRequired: true },
],
} } } };
const original = structuredClone(state);
clearActivity(root, state, 'PRIVATE-ADMIN', now);
assert.deepEqual(state, original, 'display reset changes no issue, event or ownership');
state.activityHistory = loadActivity(root);
const view = remediationView(state, { full: true, now });
assert.deepEqual(view.cases.map((c) => c.id).sort(), ['held', 'queued']);
assert.equal(view.activityHistory.hidden, 1); assert.equal(view.truncated, false);
assert.equal(view.execution, 'stalled'); assert.equal(view.stalledRuns, 1); assert.equal(view.queuedEvents, 3);
assert.doesNotMatch(JSON.stringify(remediationView(state, { now })), /PRIVATE/);
issues[0].revision++;
assert.equal(remediationView(state, { full: true, now }).cases.length, 3);
assert.equal(remediationView(state, { full: true, now }).truncated, true, 'missing new history stays visible');
writeFileSync(join(root, 'remediation-activity.json'), '{broken');
assert.equal(loadActivity(root), null, 'invalid reset must not hide anything');
});

test('administrator queue total uses receiver totals even when the case feed is truncated', () => {
const state = { issues: [], delivery: { receiver: { counts: { queued: 7 }, remediation: { cases: [], truncated: true } } } };
assert.equal(remediationView(state, { full: true }).queuedEvents, 7);
assert.equal(remediationView(state).queuedEvents, 0, 'public network filtering cannot leak private queue counts');
});
13 changes: 10 additions & 3 deletions server/remediation.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import { TITLES } from './incidents.js';
import { RELEASE_CODES } from './release-webhook.js';
import { stallReasons } from './stall-alert.js';
import { activityHidden } from './remediation-activity.js';

const fresh = (at, now) => Number.isFinite(Date.parse(at)) && Date.parse(at) <= now + 60_000 && now - Date.parse(at) <= 180_000;
const text = (value, size = 800) => typeof value === 'string' ? value.slice(0, size) : null;
Expand Down Expand Up @@ -31,7 +32,9 @@ export function remediationView(state, { full = false, visibleNetworks = [], now
const workerFresh = feed?.schemaVersion === 1 && fresh(feed.generatedAt, now) && fresh(workerAt, now) && Array.isArray(feed.cases);
const observations = new Map((Array.isArray(feed?.cases) ? feed.cases : []).map((c) => [c.issueId, c]));
const visible = new Set(visibleNetworks);
const cases = (state?.issues || []).filter((i) => full || i.domain !== 'network' || visible.has(i.scope)).map((issue) => {
const visibleIssues = (state?.issues || []).filter((i) => full || i.domain !== 'network' || visible.has(i.scope));
const displayedIssues = visibleIssues.filter((i) => !activityHidden(i, observations.get(i.id), state));
const cases = displayedIssues.map((issue) => {
const response = observations.get(issue.id);
const last = response?.lastResponse;
let stage = 'unknown', reason = 'Waiting for remediation status';
Expand Down Expand Up @@ -96,12 +99,16 @@ export function remediationView(state, { full = false, visibleNetworks = [], now
...(full ? { reset: requested?.recovered === true, changedAt: iso(requested?.updatedAt), changedBy: text(requested?.updatedBy, 39), clearedAt: iso(requested?.clearedAt), clearedBy: text(requested?.clearedBy, 39) } : {}) };
return { schemaVersion: 1, public: !full, generatedAt: iso(state?.generatedAt), workerAt, control,
stalled: attention.length > 0, attention: full ? attention : undefined, stalledSince: full ? iso(state?.stall?.since) : undefined,
stale: !producerFresh || !workerFresh, producerFresh, workerFresh, truncated: feed?.truncated === true,
stale: !producerFresh || !workerFresh, producerFresh, workerFresh,
truncated: feed?.truncated === true && displayedIssues.some((i) => !observations.has(i.id)),
activityHistory: { clearedAt: iso(state?.activityHistory?.clearedAt), hidden: visibleIssues.length - displayedIssues.length,
...(full ? { clearedBy: text(state?.activityHistory?.clearedBy, 39) } : {}) },
dispatch: !workerFresh ? 'unknown' : feed.enabled ? 'enabled' : 'paused',
execution: !workerFresh || typeof feed.progress?.attentionRequired !== 'boolean' ? 'unknown' : feed.progress.attentionRequired ? 'stalled' : 'healthy',
stalledRuns: workerFresh && Number.isSafeInteger(feed.progress?.stalledRuns) ? feed.progress.stalledRuns : null,
maxActive: Number.isSafeInteger(feed?.maxActive) ? feed.maxActive : null,
delivery: { pending: full ? state?.outbox?.length || 0 : null, failed: !!state?.delivery?.error, lastAckAt: iso(state?.delivery?.lastAckAt) },
maintenance: Object.entries(state?.maintenance || {}).filter(([name]) => full || visible.has(name)).map(([network, value]) => ({ network, operations:full ? value.operations : undefined })),
counts, queuedEvents: cases.reduce((sum, c) => sum + c.pendingEvents, 0), cases };
counts, queuedEvents: full && Number.isSafeInteger(state?.delivery?.receiver?.counts?.queued)
? state.delivery.receiver.counts.queued : cases.reduce((sum, c) => sum + c.pendingEvents, 0), cases };
}
8 changes: 7 additions & 1 deletion server/web.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { loadIncidentState, publicIssues, apiAuthorized, readSecret } from './in
import { remediationView } from './remediation.js';
import { receiveRelease } from './release-webhook.js';
import { againstReceiver, loadControl, requestClear, setDispatch } from './remediation-control.js';
import { clearActivity, loadActivity } from './remediation-activity.js';
import { publicInventory } from './aws-public.js';
import { COMPONENTS, COMPONENT_REPOS, accessFor, adminFor, loadSettings, memberOf, operatorFor, readJSON, saveSettings, validateSettings, writeAtomic } from '../shared/settings.js';
import { expectations } from '../shared/monitoring.js';
Expand Down Expand Up @@ -142,7 +143,7 @@ export function createWeb({ dataDir, origin, auth: authDeps, fetcher = fetch, cl
reloadSettings();
const full = isAdmin(req) || apiAuthorized(req.get('authorization'), readSecret(process.env.INCIDENT_API_TOKEN_FILE));
const incidents = loadIncidentState(dataDir);
res.json(remediationView({ ...incidents, maintenance:operationMaintenance(dataDir, settings, clock()), control: againstReceiver(loadControl(dataDir), incidents?.delivery?.receiver?.control) }, { full, visibleNetworks: visible(req).map((n) => n.name), now: clock() }));
res.json(remediationView({ ...incidents, activityHistory: loadActivity(dataDir), maintenance:operationMaintenance(dataDir, settings, clock()), control: againstReceiver(loadControl(dataDir), incidents?.delivery?.receiver?.control) }, { full, visibleNetworks: visible(req).map((n) => n.name), now: clock() }));
});
app.get('/api/me', (req, res) => {
const u = user(req);
Expand Down Expand Up @@ -388,6 +389,11 @@ export function createWeb({ dataDir, origin, auth: authDeps, fetcher = fetch, cl
controlled(res, setDispatch(dataDir, req.body.enabled, req.session.user.login, clock(), acknowledged()));
});
app.post('/api/remediation/clear', ...adminWrite, (req, res) => controlled(res, requestClear(dataDir, req.session.user.login, clock(), acknowledged())));
app.post('/api/remediation/activity/clear', ...adminWrite, (req, res) => {
const activityHistory = clearActivity(dataDir, loadIncidentState(dataDir), req.session.user.login, clock());
push('remediation', { at: new Date(clock()).toISOString() });
res.json({ activityHistory });
});

// Infrastructure pages: full detail for any account with access; before
// sign-in a sanitized view (statistics and spend, nothing identifying).
Expand Down
9 changes: 9 additions & 0 deletions server/web.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -317,9 +317,11 @@ test('remediation switch and queue clear are administrator-only, CSRF-protected
try {
assert.equal((await post(w, '/api/remediation/dispatch', { enabled: true })).status, 401);
assert.equal((await post(w, '/api/remediation/clear')).status, 401);
assert.equal((await post(w, '/api/remediation/activity/clear')).status, 401);
const csrf = await w.login();
assert.equal((await (await w.req('/api/remediation')).json()).control.requested, 'off');
assert.equal((await post(w, '/api/remediation/dispatch', { enabled: true })).status, 403, 'a session without the CSRF token is refused');
assert.equal((await post(w, '/api/remediation/activity/clear')).status, 403);
assert.equal((await post(w, '/api/remediation/dispatch', { enabled: 'yes' }, csrf)).status, 400);
assert.equal((await post(w, '/api/remediation/dispatch', { enabled: true }, csrf)).status, 200);
assert.deepEqual([stored(w).seq, stored(w).dispatch, stored(w).updatedBy], [1, 'on', 'someone']);
Expand All @@ -342,12 +344,19 @@ test('remediation switch and queue clear are administrator-only, CSRF-protected
assert.equal((await post(w, '/api/remediation/clear', undefined, csrf)).status, 200);
assert.deepEqual([stored(w).seq, stored(w).dispatch, stored(w).clearSeq], [11, 'off', 11], 'a clear after the loss is not stale either, and does not switch on');
assert.equal((await (await w.req('/api/remediation')).json()).control.reset, false);
const beforeHistoryClear = stored(w);
const beforeIssues = readFileSync(join(w.dataDir, 'incidents', 'state.json'), 'utf8');
assert.equal((await post(w, '/api/remediation/activity/clear', undefined, csrf)).status, 200);
assert.deepEqual(stored(w), beforeHistoryClear, 'activity clear never changes dispatch or the queue');
assert.equal(readFileSync(join(w.dataDir, 'incidents', 'state.json'), 'utf8'), beforeIssues);
assert.equal((await (await w.req('/api/remediation')).json()).activityHistory.clearedBy, 'someone');
} finally { w.close(); }
const member = await start(42);
try {
const csrf = await member.login();
assert.equal((await post(member, '/api/remediation/dispatch', { enabled: true }, csrf)).status, 403);
assert.equal((await post(member, '/api/remediation/clear', undefined, csrf)).status, 403);
assert.equal((await post(member, '/api/remediation/activity/clear', undefined, csrf)).status, 403);
assert.equal((await (await member.req('/api/remediation')).json()).control.changedBy, undefined);
} finally { member.close(); }
});
Expand Down
3 changes: 3 additions & 0 deletions src/pages/Remediation.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ export default function Remediation() {
</div>
<p className="text-dim text-[11px] mt-2">Counts are unique issues. {data.queuedEvents} queued events may include repeat updates. A completed investigation is not a verified fix.</p>
<Section title="Issue activity" right={<span className="text-dim text-[12px]">{cases.length} shown</span>}>
{data.activityHistory?.clearedAt && <p className="text-dim text-[12px] mb-3">Activity cleared {ago(data.activityHistory.clearedAt, now)} ago · {data.activityHistory.hidden} older entries hidden. New events and queued or active repairs remain visible. Monitoring findings are unchanged.</p>}
<div className="flex flex-wrap gap-2 items-center mb-3">
<label className="sr-only" htmlFor="remediation-search">Filter remediation issues</label>
<input id="remediation-search" className="input flex-1 min-w-40" placeholder="Filter issues…" value={search} onChange={(e) => setSearch(e.target.value)} />
Expand Down Expand Up @@ -91,6 +92,8 @@ function Controls({ data, now, reload }) {
: `Switch autonomous repair on? ${data.queuedEvents} queued event(s) become eligible for the repair agent.`)}>{on ? 'Switch off' : 'Switch on'}</button>
<button className="btn btn-danger" disabled={busy || !c.supported}
onClick={() => run('/api/remediation/clear', undefined, `Clear the repair queue? ${data.queuedEvents} queued event(s) are removed without being repaired. Repairs in progress and release compatibility tasks are not affected. Issues that are still open return on their next monitoring update.`)}>Clear queue</button>
<button className="btn" disabled={busy}
onClick={() => run('/api/remediation/activity/clear', undefined, 'Clear the displayed issue activity? Older entries are hidden without marking issues fixed. Queued work and active or held repairs remain visible; new events appear normally.')}>Clear activity</button>
<span className="text-dim">{!c.supported ? 'The repair worker has not announced these controls yet; they become available once it runs this release.'
: c.applied == null ? 'Worker acknowledgement is unavailable; the requested setting is not yet confirmed.'
: c.pending || c.applied !== c.requested ? 'Waiting for the repair worker to acknowledge (about a minute).' : 'Acknowledged by the repair worker.'}</span>
Expand Down
Loading