From 501f02be316f7f4da9374a64967a0cf2da73ab8f Mon Sep 17 00:00:00 2001 From: knqiufan Date: Tue, 13 Oct 2026 10:14:26 +0800 Subject: [PATCH 1/6] fix(topic-memory): add durable block inspection and recovery --- .../src-tauri/src/transport/operations.json | 2 +- desktop/ui/src/generated/api.d.ts | 198 ++++++++++ desktop/ui/src/generated/operations.ts | 2 +- docs/en/docs/operate/troubleshoot.md | 9 +- docs/en/docs/workflows/topic-memory.md | 37 ++ docs/en/rfcs/1417_topic_memory.md | 27 +- .../1515_artifact_processing_supervisor.md | 6 + docs/zh/docs/operate/troubleshoot.md | 9 +- docs/zh/docs/workflows/topic-memory.md | 32 ++ docs/zh/rfcs/1417_topic_memory.md | 19 +- .../1515_artifact_processing_supervisor.md | 5 + .../powercontext/src/operations.generated.ts | 3 + .../powercontext/src/operations.generated.ts | 3 + .../powercontext/src/operations.generated.ts | 3 + openapi/powercontext.yaml | 158 ++++++++ .../artifacts/topic_memory/processing.py | 61 ++++ .../builtin/persistence/tables.py | 34 ++ .../persistence/topic_memory_budget.py | 157 +++++++- .../persistence/topic_memory_recovery.py | 224 ++++++++++++ .../builtin/runtime/application.py | 47 +++ .../builtin/runtime/artifact_processing.py | 29 ++ .../builtin/runtime/composition.py | 4 + .../runtime/topic_memory_processing.py | 3 + .../builtin/runtime/topic_memory_recovery.py | 84 +++++ .../builtin/runtime/topic_memory_scope.py | 45 ++- src/powercontext/client/cli.py | 69 ++++ src/powercontext/client/client.py | 27 ++ src/powercontext/http/__init__.py | 16 + src/powercontext/http/_generated/models.py | 73 ++++ .../http/_generated/operations.py | 87 +++++ src/powercontext/http/_generated/schema.py | 238 ++++++++++++ src/powercontext/server/app.py | 67 ++++ .../runtime/test_topic_memory_recovery.py | 339 ++++++++++++++++++ .../e2e/test_topic_memory_recovery_access.py | 137 +++++++ .../test_topic_memory_terminal_frontier.py | 118 +++++- tests/test_topic_memory_recovery_cli.py | 95 +++++ 36 files changed, 2389 insertions(+), 78 deletions(-) create mode 100644 src/powercontext/builtin/artifacts/topic_memory/processing.py create mode 100644 src/powercontext/builtin/persistence/topic_memory_recovery.py create mode 100644 src/powercontext/builtin/runtime/topic_memory_recovery.py create mode 100644 tests/builtin/runtime/test_topic_memory_recovery.py create mode 100644 tests/e2e/test_topic_memory_recovery_access.py create mode 100644 tests/test_topic_memory_recovery_cli.py diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index b796694802..86cbf43a2a 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "c4571e4bd59833451e3091ee604e836d185643b0b0d9fa08dceb8eea43060cbf", + "contractSha256": "5e0de479c92d57c2286fe04f3ee2c8e3321798f35412fc237920c105df48ed2d", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index 5be80d4d27..9a855a1809 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -838,6 +838,66 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/topic-memory/processing/status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Inspect Topic Memory processing and its current terminal block + * @description Return the current Cursor, Source head, retained target and durable blocked-window diagnostics. An active final attempt remains pending because it may still finish. Legacy blocks receive their first observation timestamp; an unavailable historical error code is null. This operation never grants new processing allowance. + */ + post: operations["get_topic_memory_processing_status"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/topic-memory/processing/blocks/get": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Get a retained Topic Memory processing block + * @description Return one exact block, including its original diagnostics and any accepted retry, after recovery. + */ + post: operations["get_topic_memory_processing_block"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/topic-memory/processing/retry": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Explicitly retry one observed Topic Memory block + * @description Require scope.admin and the exact block_id returned by status. Atomically retain the block's failure and usage evidence, grant one fresh bounded allowance and request asynchronous processing without advancing the Cursor or discarding the target or Sources. Repeating a block_id returns its original acceptance and cannot reset a later attempt. An unretried block that no longer owns the frontier returns topic_memory_block_conflict. Ordinary flushes, status reads, restarts and model changes never reset allowance. + */ + post: operations["retry_topic_memory_processing"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/topic-memory/search": { parameters: { query?: never; @@ -3761,6 +3821,51 @@ export interface components { */ hold_codes: string[]; }; + TopicMemoryProcessingStatusRequest: { + scope_id: string; + }; + GetTopicMemoryProcessingBlockRequest: { + scope_id: string; + block_id: string; + }; + RetryTopicMemoryProcessingRequest: { + scope_id: string; + block_id: string; + }; + TopicMemoryProcessingBlock: { + block_id: string; + attempt_id: string; + scope_id: string; + binding_name: string; + source_after: number; + source_through: number; + target_source_through: number; + attempts: number; + requests: number; + tokens: number; + error_code: string; + last_error_code: string | null; + /** Format: date-time */ + blocked_at: string; + /** Format: date-time */ + retried_at: string | null; + retry_request_generation: number | null; + }; + /** @enum {string} */ + TopicMemoryProcessingState: "idle" | "pending" | "blocked"; + /** @enum {string} */ + TopicMemoryProcessingRetryStatus: "accepted"; + TopicMemoryProcessingStatus: { + state: components["schemas"]["TopicMemoryProcessingState"]; + source_cursor: number; + source_head: number; + target_source_through: number | null; + block: components["schemas"]["TopicMemoryProcessingBlock"] | null; + }; + RetryTopicMemoryProcessingResponse: { + status: components["schemas"]["TopicMemoryProcessingRetryStatus"]; + block: components["schemas"]["TopicMemoryProcessingBlock"]; + }; FlushTopicMemoryRequest: { scope_id: string; }; @@ -6500,6 +6605,99 @@ export interface operations { 503: components["responses"]["Unavailable"]; }; }; + get_topic_memory_processing_status: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["TopicMemoryProcessingStatusRequest"]; + }; + }; + responses: { + /** @description Current durable processing state. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TopicMemoryProcessingStatus"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; + get_topic_memory_processing_block: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GetTopicMemoryProcessingBlockRequest"]; + }; + }; + responses: { + /** @description The exact retained block. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TopicMemoryProcessingBlock"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; + retry_topic_memory_processing: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RetryTopicMemoryProcessingRequest"]; + }; + }; + responses: { + /** @description One retry was durably accepted, including duplicate acceptance. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RetryTopicMemoryProcessingResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; search_topic_memory: { parameters: { query?: never; diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index b2d1479877..ca86cf8c45 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "c4571e4bd59833451e3091ee604e836d185643b0b0d9fa08dceb8eea43060cbf"; +export const contractSha256 = "5e0de479c92d57c2286fe04f3ee2c8e3321798f35412fc237920c105df48ed2d"; export const operations = { "get_liveness": { "method": "GET", diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 31acac62a1..a6fa38b5e8 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -259,16 +259,17 @@ so the previous database remains available for recovery: Because `pc_scopes.parent_scope_id` is self-referential, keep ancestor Scope rows before their descendants in the exported `pc_scopes` data. If the source predates the three Skill lifecycle tables (`pc_skill_packages`, `pc_agent_skill_targets`, and - `pc_skill_publications`), the Profile tables, `pc_topic_memory_work_budgets`, `pc_recall_effort_daily`, - `pc_receipt_migration_review`, `pc_context_bootstrap_receipts`, or `pc_decision_observations`, remove the absent - tables from their respective layers. + `pc_skill_publications`), the Profile tables, `pc_topic_memory_work_budgets`, `pc_topic_memory_work_attempts`, + `pc_recall_effort_daily`, `pc_receipt_migration_review`, `pc_context_bootstrap_receipts`, or + `pc_decision_observations`, remove the absent tables from their respective layers. When a work-budget table exists, restore it together with Cursors so failure allowances survive the migration. + Restore work-attempt records as well to preserve block identities, diagnostics and accepted retry receipts. Layer 1 contains parents and tables without foreign keys: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_attempts,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \ -f ``` diff --git a/docs/en/docs/workflows/topic-memory.md b/docs/en/docs/workflows/topic-memory.md index 0050afaa01..69365b320a 100644 --- a/docs/en/docs/workflows/topic-memory.md +++ b/docs/en/docs/workflows/topic-memory.md @@ -63,6 +63,43 @@ with a Generation model require a persistent, file-backed database rather than a [Configure models and full memory](../get-started/configure-models.md) and [Configuration options](../operate/configuration.md) for the complete settings and deployment constraints. +## Inspect and retry blocked processing + +After processing exhausts its allowance, the same Scope's later Sources remain pending. The Supervisor stops launching +Workers for that frontier and periodically checks metadata. Other Scopes can continue. A flush or restart does not +grant more allowance. + +Inspect the current Cursor, Source head, target and block with `scope.read` permission: + +```bash +powercontext --json topic-memory status --scope-id "$SCOPE_ID" +``` + +The state is `idle`, `pending`, or `blocked`. A block contains its exact `block_id`, attempted range, retained target, +attempt/request/token counters, terminal `error_code`, `last_error_code`, and UTC `blocked_at`. An active third attempt +remains pending because it can still finish. For legacy budgets, `blocked_at` is the first observation time and an +unknown last error is null. + +Correct the cause, then copy the returned block ID into `BLOCK_ID`. A Scope administrator can explicitly grant one +new bounded allowance and request processing: + +```bash +powercontext --json topic-memory retry --scope-id "$SCOPE_ID" --block-id "$BLOCK_ID" +powercontext --json topic-memory block --scope-id "$SCOPE_ID" --block-id "$BLOCK_ID" +``` + +Retry requires `scope.admin`. Its `accepted` response means the request committed; it does not promise generation +success. These reads require `scope.read`, which the `scope.admin` role does not include. Repeating the same ID returns +the original acceptance even after processing is disabled, and cannot reset a later attempt. An unretried block +that no longer owns the current frontier returns HTTP 409. Cursor, target and Sources are retained; the original block +and its retry timestamp remain readable after progress resumes. Normal flushes and restarts retain their existing +budget behavior. Each explicitly accepted retry grants the same three-attempt and provider ceilings. + +The equivalent Python Client SDK methods are `get_topic_memory_processing_status`, +`get_topic_memory_processing_block`, and `retry_topic_memory_processing`. Their HTTP routes are +`POST /v1/topic-memory/processing/status`, `POST /v1/topic-memory/processing/blocks/get`, and +`POST /v1/topic-memory/processing/retry`. These administrative operations are not MCP tools. + ## Verify a real topic Use the [Quick Start Topic Memory check](../get-started/quickstart.md#4-verify-topic-memory-with-ordinary-conversation) diff --git a/docs/en/rfcs/1417_topic_memory.md b/docs/en/rfcs/1417_topic_memory.md index 6e454a1de9..0bbe36e498 100644 --- a/docs/en/rfcs/1417_topic_memory.md +++ b/docs/en/rfcs/1417_topic_memory.md @@ -747,17 +747,30 @@ Supervisor's shared admission time allowance yields control without attributing A terminal frontier retains its Sources, Cursor, Pending, and later same-Scope Sources. It is not a NOOP, success, or permission to skip evidence. Other Scope keys remain processable. A retry that succeeds within the remaining allowance -publishes normally and can process the tail. There is no automatic allowance reset, retry/reset API, or quarantine -skip operation; terminal input or repeated failures require deliberate operator remediation. Repeated flushes and -restarts cannot authorize additional model cost. +publishes normally and can process the tail. There is no automatic allowance reset or quarantine skip operation. +`POST /v1/topic-memory/processing/status` exposes the current Cursor, Source head, target and block; an active final +attempt remains pending. `POST /v1/topic-memory/processing/blocks/get` reads an exact retained block. Scope readers may +inspect these records; `POST /v1/topic-memory/processing/retry` requires `scope.admin` and the observed block ID. It +atomically retains the old evidence, grants one new bounded allowance and requests processing without advancing the +Cursor or deleting the target or Sources. Duplicate IDs return the original acceptance, even if another attempt has +started or progress advanced. A stale unretried ID returns a conflict. Repeated ordinary flushes and restarts cannot +authorize additional model cost. + +`pc_topic_memory_work_attempts` keeps one metadata record per attempt, including its exact owner, attempted range, +target, usage snapshot and last error category. A terminal observation adds a stable block ID and `blocked_at`; retry +records its timestamp and accepted request generation without replacing the original diagnostics. Records contain no +Source/model text. The additive table is created on supported database startup without altering the budget schema. +Legacy rows receive an observation time and nullable historical error; no original failure timestamp is inferred. +Lookups use indexed identities and writes touch one frontier. Normal processing adds at most three attempt records +per window allowance; explicitly accepted retries add another bounded allowance. Records are retained for audit. `pc_topic_memory_work_budgets` stores the frontier, furthest attempted end, opaque attempt ID, attempt count, reserved requests/tokens, and a bounded failure code; it stores no prompts, Source text, or model outputs. Inspect these columns together with the Cursor and structured error logs to distinguish terminal work from transient backoff. A new attempt -supersedes publication authority from an earlier attempt in the same leadership term. The row is removed only in the -successful atomic publication transaction alongside Cursor CAS and Topic/index writes; rollback restores it. No -fragment checkpoint or persistent job history is created. The table is additive and is created when an existing -supported database opens. +supersedes publication authority from an earlier attempt in the same leadership term. Successful publication removes +the row atomically alongside Cursor CAS and Topic/index writes; rollback restores it. An explicitly accepted retry +also removes the exact exhausted allowance atomically with its retained audit record and new request. No fragment +checkpoint is created. The tables are additive and are created when an existing supported database opens. Actual errors—including model calls, output validation, retrieval, enabled Embedding, database commit, Worker crash, and timeout—use the same backoff strategy but must produce structured logs by `stage` and `error_code`. Cursor/Head CAS diff --git a/docs/en/rfcs/1515_artifact_processing_supervisor.md b/docs/en/rfcs/1515_artifact_processing_supervisor.md index 350e854a80..f8c3df37a0 100644 --- a/docs/en/rfcs/1515_artifact_processing_supervisor.md +++ b/docs/en/rfcs/1515_artifact_processing_supervisor.md @@ -520,6 +520,12 @@ The attempted request generation is remembered even when the check fails, so ord the interval, while a newer explicit request can still recheck promptly. A probe's own timeout, distinct from shared allowance expiry, remains a genuine probe error. +A binding may also register a metadata-only `work_failure` callback. After a failed Worker is stopped, the current +fenced Supervisor calls it with the Scope, binding and error category, so a domain can retain crash/timeout diagnostics +without changing allowance. The callback has a bounded I/O timeout; recording failure is warned about without losing +the original failure or stopping other keys. Topic processing retains an indexed attempt/block record and exposes +Scope-authorized inspection and explicit retry through its domain APIs; the Supervisor does not reset budgets. + Memory owns a persisted per-Scope window reduction for generation timeouts. A failed extraction can halve the next window without advancing its Source cursor or acknowledging the invocation; the reduction survives Worker replacement and is cleared when the observed backlog is consumed. diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index d3102f929f..d5f2c445ad 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -250,16 +250,17 @@ collation，但不会包含数据库 URL 或凭据。 删除它。由于 `pc_scopes.parent_scope_id` 自引用 `pc_scopes`，导出的 `pc_scopes` 数据必须让祖先 Scope 记录排在 后代记录之前。 源数据库早于三张 Skill 生命周期表（`pc_skill_packages`、`pc_agent_skill_targets` 和 - `pc_skill_publications`）、Profile 表、`pc_topic_memory_work_budgets`、`pc_recall_effort_daily`、 - `pc_receipt_migration_review`、`pc_context_bootstrap_receipts` 或 `pc_decision_observations` 时，应从对应层 - 删除缺失的表。 + `pc_skill_publications`）、Profile 表、`pc_topic_memory_work_budgets`、`pc_topic_memory_work_attempts`、 + `pc_recall_effort_daily`、`pc_receipt_migration_review`、`pc_context_bootstrap_receipts` 或 + `pc_decision_observations` 时，应从对应层删除缺失的表。 如果存在 work-budget 表，必须与 Cursor 一起恢复，保留已消耗的失败额度，避免迁移后重置累计成本。 + work-attempt 表也应一起恢复，保留阻塞身份、诊断信息与已接受的重试回执。 第 1 层包含父表和无外键的表： ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_attempts,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations' \ -f ``` diff --git a/docs/zh/docs/workflows/topic-memory.md b/docs/zh/docs/workflows/topic-memory.md index ac826b7c5f..bd276ee360 100644 --- a/docs/zh/docs/workflows/topic-memory.md +++ b/docs/zh/docs/workflows/topic-memory.md @@ -85,6 +85,38 @@ Content-Type: application/json 响应只包含 `status`。Source capture、主题生成和索引更新由部署配置的 worker 异步完成；没有配置生成模型或后台 处理能力时，flush 不会凭空产生主题。向量或混合检索是否可用也是部署选项，不由调用方在请求中指定。 +## 查询并恢复阻塞的处理 + +处理额度耗尽后，同一 Scope 后续的 Source 会保留待处理。Supervisor 停止为该 frontier 启动 Worker，仅定期检查 +metadata，其他 Scope 可继续处理。普通 flush 和重启不会增加额度。 + +具有 `scope.read` 权限时，可以查询当前 Cursor、Source head、target 和阻塞记录： + +```bash +powercontext --json topic-memory status --scope-id "$SCOPE_ID" +``` + +状态为 `idle`、`pending` 或 `blocked`。阻塞记录包含准确的 `block_id`、已尝试范围、保留的 target、尝试/请求/token +计数、终止原因 `error_code`、最后错误类别 `last_error_code` 和 UTC 时间 `blocked_at`。仍在执行的第三次尝试可能 +成功，因此显示为 pending。旧预算的 `blocked_at` 表示首次观察到阻塞的时间，无法追溯的最后错误类别为 null。 + +修复失败原因后，将返回的阻塞 ID 填入 `BLOCK_ID`。Scope 管理员可显式申请一次新的有限额度，并请求后台处理： + +```bash +powercontext --json topic-memory retry --scope-id "$SCOPE_ID" --block-id "$BLOCK_ID" +powercontext --json topic-memory block --scope-id "$SCOPE_ID" --block-id "$BLOCK_ID" +``` + +retry 需要 `scope.admin` 权限，查询还需要该角色未包含的 `scope.read` 权限。`accepted` 表示请求已提交，不保证生成成功。 +重复提交同一个 ID 只返回原接受结果，即使后台处理已关闭也如此， +不会重置后续尝试；尚未重试且已不再对应当前 frontier 的记录返回 HTTP 409。Cursor、target 和 Source 均保留， +恢复后仍可读取原阻塞记录和重试时间。普通 flush、重启的额度语义不变；每次明确接受的 retry 使用相同的三次尝试 +和 provider 上限。 + +Python Client SDK 对应方法为 `get_topic_memory_processing_status`、`get_topic_memory_processing_block` 和 +`retry_topic_memory_processing`。HTTP 路径分别为 `POST /v1/topic-memory/processing/status`、 +`POST /v1/topic-memory/processing/blocks/get` 和 `POST /v1/topic-memory/processing/retry`。这些管理操作不作为 MCP 工具提供。 + ## 搜索当前主题 搜索需要 `scope.read` 权限，输入 `scope_id`、非空 `query`，可选 `limit`（范围 1–20，默认 10）： diff --git a/docs/zh/rfcs/1417_topic_memory.md b/docs/zh/rfcs/1417_topic_memory.md index e6a5df3a4a..8716c3071c 100644 --- a/docs/zh/rfcs/1417_topic_memory.md +++ b/docs/zh/rfcs/1417_topic_memory.md @@ -658,13 +658,24 @@ metadata 重新检查出错时保留已知终止状态、等待间隔和日志 不将正在检查的 Scope 记成新的失败。 终止的 frontier 保留 Source、Cursor、Pending 以及同 Scope 尾部；不会视为 NOOP、成功或跳过证据的授权。其他 Scope -仍可处理。剩余额度内重试成功后仍按原合同发布，并继续处理尾部。不提供自动额度重置、retry/reset API 或 quarantine -跳过操作；终止输入或重复失败需要运维明确修复，反复 flush 或重启不能重新授权模型成本。 +仍可处理。剩余额度内重试成功后仍按原合同发布，并继续处理尾部。不提供自动额度重置或 quarantine 跳过操作。 +`POST /v1/topic-memory/processing/status` 返回当前 Cursor、Source head、target 和阻塞记录；仍在执行的最后一次尝试 +显示为 pending。`POST /v1/topic-memory/processing/blocks/get` 读取准确的历史阻塞记录。这些查询需要 `scope.read`。 +`POST /v1/topic-memory/processing/retry` 需要 `scope.admin` 和查询得到的 block ID，在一个事务中保留旧诊断、申请一次 +新的有限额度并接受处理请求，不推进 Cursor、不删除 target 或 Source。重复 ID 返回原接受结果，不能重置后续尝试； +尚未重试且已过期的 ID 返回冲突。普通 flush 和重启仍不能重新授权模型成本。 + +`pc_topic_memory_work_attempts` 为每次尝试保留一条 metadata 记录，包含准确的执行 owner、已尝试范围、target、额度 +快照及最后错误类别。首次观察到终止时记录稳定 block ID 和 `blocked_at`；retry 保存重试时间及接受的请求 generation， +不覆盖原诊断。记录不含 Source 或模型原文。正常启动会创建新增表，不修改旧预算表。旧记录使用首次观察时间，缺失的 +历史错误类别为 null，不推测原始失败时间。查询使用索引身份，写入仅作用于一个 frontier；正常每个窗口额度最多产生 +三条尝试记录，显式接受的 retry 可再申请相同的有限额度。记录保留用于审计。 `pc_topic_memory_work_budgets` 保存 frontier、尝试过的最远终点、不透明 attempt ID、尝试次数、预留请求/token 与有界 failure code，不保存 prompt、Source 原文或模型输出。可结合 Cursor 和结构化错误日志检查这些列，区分持久终止与暂时 -退避。同一任期内新尝试也会使旧尝试失去发布权。只有成功原子发布事务才能与 Cursor CAS、Topic/index 写入一起删除 -该记录；回滚会恢复额度记录。不创建分片 checkpoint 或持久 Job history。该新增表会在现有受支持数据库打开时创建。 +退避。同一任期内新尝试也会使旧尝试失去发布权。成功原子发布事务与 Cursor CAS、Topic/index 写入一起删除额度记录， +回滚会恢复记录。显式接受的 retry 也可在一个事务中移除准确的耗尽额度，并保留旧审计记录与新的处理请求。不创建 +分片 checkpoint。新增表会在现有受支持数据库打开时创建。 模型调用、输出校验、检索、启用后的 Embedding、数据库提交、Worker crash 和 timeout 等实际错误采用同一退避策略， 但必须按 `stage` 和 `error_code` 写结构化日志。Cursor/Head CAS 冲突与 leadership lost 是控制信号，不增加 diff --git a/docs/zh/rfcs/1515_artifact_processing_supervisor.md b/docs/zh/rfcs/1515_artifact_processing_supervisor.md index ab2d1d90dc..2aa7d66544 100644 --- a/docs/zh/rfcs/1515_artifact_processing_supervisor.md +++ b/docs/zh/rfcs/1515_artifact_processing_supervisor.md @@ -425,6 +425,11 @@ Worker 保留自己的领域检查；准入后遇到持久拒绝时，通过脱 普通唤醒不能反复绕过间隔，更新的显式请求仍可及时重新检查。probe 自身抛出的超时与共享额度耗尽区分，按实际检查 错误处理。 +binding 还可注册仅记录 metadata 的 `work_failure` 回调。失败 Worker 已停止后，当前任期内的 Supervisor 校验 fencing， +将 Scope、binding 和错误类别交给领域保存 crash/timeout 诊断，不改变额度。该回调具有有界 I/O 超时；记录失败时输出 +warning，仍保留原处理错误并让其他键继续运行。Topic 领域保留带索引的尝试/阻塞记录，通过自身 API 提供具有 Scope +权限的查询和显式 retry；Supervisor 不负责重置预算。 + Memory 在生成超时后按 Scope 持久化缩窗上限。失败的提取可以将下一次窗口减半，但不推进 Source 游标， 也不确认调用；缩窗上限跨 Worker 更换保留，在已观察到的积压消费完成时清除。该更新受游标 CAS 和当前 任期 fencing 保护。它是输入大小提示，不代表处理成功的 checkpoint，也不是 Supervisor 的失败任务状态。 diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index 104f93c7ed..f3b84a0cc3 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -64,6 +64,9 @@ export const OPERATIONS = { prepare_handoff_hint: { method: 'POST', path: '/v1/handoff/hint', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_topic_memory: { method: 'POST', path: '/v1/topic-memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_artifacts: { method: 'POST', path: '/v1/scopes/{scope_id}/artifacts/{family}/search', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'family'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_topic_memory_processing_status: { method: 'POST', path: '/v1/topic-memory/processing/status', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_topic_memory_processing_block: { method: 'POST', path: '/v1/topic-memory/processing/blocks/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + retry_topic_memory_processing: { method: 'POST', path: '/v1/topic-memory/processing/retry', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_topic_memory: { method: 'POST', path: '/v1/topic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_topic_memory: { method: 'POST', path: '/v1/topic-memory/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_memory: { method: 'POST', path: '/v1/memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index 104f93c7ed..f3b84a0cc3 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -64,6 +64,9 @@ export const OPERATIONS = { prepare_handoff_hint: { method: 'POST', path: '/v1/handoff/hint', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_topic_memory: { method: 'POST', path: '/v1/topic-memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_artifacts: { method: 'POST', path: '/v1/scopes/{scope_id}/artifacts/{family}/search', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'family'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_topic_memory_processing_status: { method: 'POST', path: '/v1/topic-memory/processing/status', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_topic_memory_processing_block: { method: 'POST', path: '/v1/topic-memory/processing/blocks/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + retry_topic_memory_processing: { method: 'POST', path: '/v1/topic-memory/processing/retry', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_topic_memory: { method: 'POST', path: '/v1/topic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_topic_memory: { method: 'POST', path: '/v1/topic-memory/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_memory: { method: 'POST', path: '/v1/memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index 104f93c7ed..f3b84a0cc3 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -64,6 +64,9 @@ export const OPERATIONS = { prepare_handoff_hint: { method: 'POST', path: '/v1/handoff/hint', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_topic_memory: { method: 'POST', path: '/v1/topic-memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_artifacts: { method: 'POST', path: '/v1/scopes/{scope_id}/artifacts/{family}/search', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'family'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_topic_memory_processing_status: { method: 'POST', path: '/v1/topic-memory/processing/status', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_topic_memory_processing_block: { method: 'POST', path: '/v1/topic-memory/processing/blocks/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + retry_topic_memory_processing: { method: 'POST', path: '/v1/topic-memory/processing/retry', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_topic_memory: { method: 'POST', path: '/v1/topic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_topic_memory: { method: 'POST', path: '/v1/topic-memory/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_memory: { method: 'POST', path: '/v1/memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index dcb2378621..894f1a3ca0 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -1820,6 +1820,98 @@ paths: $ref: "#/components/responses/Unavailable" "500": $ref: "#/components/responses/InternalError" + /v1/topic-memory/processing/status: + post: + tags: [topic-memory] + summary: Inspect Topic Memory processing and its current terminal block + description: >- + Return the current Cursor, Source head, retained target and durable blocked-window diagnostics. An active final + attempt remains pending because it may still finish. Legacy blocks receive their first observation timestamp; + an unavailable historical error code is null. This operation never grants new processing allowance. + operationId: get_topic_memory_processing_status + x-powercontext-access: {action: scope.read, resource: {type: scope, scope-id-from: scope_id}} + x-powercontext-scope-mode: current + requestBody: + required: true + content: + application/json: + schema: {$ref: "#/components/schemas/TopicMemoryProcessingStatusRequest"} + responses: + "200": + description: Current durable processing state. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: {$ref: "#/components/schemas/TopicMemoryProcessingStatus"} + "401": {$ref: "#/components/responses/Unauthorized"} + "403": {$ref: "#/components/responses/Forbidden"} + "422": {$ref: "#/components/responses/InvalidRequest"} + "503": {$ref: "#/components/responses/Unavailable"} + "500": {$ref: "#/components/responses/InternalError"} + /v1/topic-memory/processing/blocks/get: + post: + tags: [topic-memory] + summary: Get a retained Topic Memory processing block + description: Return one exact block, including its original diagnostics and any accepted retry, after recovery. + operationId: get_topic_memory_processing_block + x-powercontext-access: {action: scope.read, resource: {type: scope, scope-id-from: scope_id}} + x-powercontext-scope-mode: current + requestBody: + required: true + content: + application/json: + schema: {$ref: "#/components/schemas/GetTopicMemoryProcessingBlockRequest"} + responses: + "200": + description: The exact retained block. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: {$ref: "#/components/schemas/TopicMemoryProcessingBlock"} + "401": {$ref: "#/components/responses/Unauthorized"} + "403": {$ref: "#/components/responses/Forbidden"} + "404": {$ref: "#/components/responses/NotFound"} + "422": {$ref: "#/components/responses/InvalidRequest"} + "503": {$ref: "#/components/responses/Unavailable"} + "500": {$ref: "#/components/responses/InternalError"} + /v1/topic-memory/processing/retry: + post: + tags: [topic-memory] + summary: Explicitly retry one observed Topic Memory block + description: >- + Require scope.admin and the exact block_id returned by status. Atomically retain the block's failure and usage + evidence, grant one fresh bounded allowance and request asynchronous processing without advancing the Cursor or + discarding the target or Sources. Repeating a block_id returns its original acceptance and cannot reset a later + attempt. An unretried block that no longer owns the frontier returns topic_memory_block_conflict. Ordinary + flushes, status reads, restarts and model changes never reset allowance. + operationId: retry_topic_memory_processing + x-powercontext-access: {action: scope.admin, resource: {type: scope, scope-id-from: scope_id}} + x-powercontext-scope-mode: current + requestBody: + required: true + content: + application/json: + schema: {$ref: "#/components/schemas/RetryTopicMemoryProcessingRequest"} + responses: + "200": + description: One retry was durably accepted, including duplicate acceptance. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: {$ref: "#/components/schemas/RetryTopicMemoryProcessingResponse"} + "401": {$ref: "#/components/responses/Unauthorized"} + "403": {$ref: "#/components/responses/Forbidden"} + "404": {$ref: "#/components/responses/NotFound"} + "409": {$ref: "#/components/responses/Conflict"} + "422": {$ref: "#/components/responses/InvalidRequest"} + "503": {$ref: "#/components/responses/Unavailable"} + "500": {$ref: "#/components/responses/InternalError"} /v1/topic-memory/search: post: tags: [topic-memory] @@ -9417,6 +9509,72 @@ components: type: string default: [] description: Compatibility field; Atomic Memory processing always returns an empty array. + TopicMemoryProcessingStatusRequest: + type: object + additionalProperties: false + required: [scope_id] + properties: + scope_id: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + GetTopicMemoryProcessingBlockRequest: + type: object + additionalProperties: false + required: [scope_id, block_id] + properties: + scope_id: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + block_id: {type: string, minLength: 1, maxLength: 36} + RetryTopicMemoryProcessingRequest: + type: object + additionalProperties: false + required: [scope_id, block_id] + properties: + scope_id: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + block_id: {type: string, minLength: 1, maxLength: 36} + TopicMemoryProcessingBlock: + type: object + additionalProperties: false + required: [block_id, attempt_id, scope_id, binding_name, source_after, source_through, target_source_through, attempts, requests, tokens, error_code, last_error_code, blocked_at, retried_at, retry_request_generation] + properties: + block_id: {type: string} + attempt_id: {type: string} + scope_id: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + binding_name: {type: string} + source_after: {type: integer, minimum: 0} + source_through: {type: integer, minimum: 1} + target_source_through: {type: integer, minimum: 1} + attempts: {type: integer, minimum: 1} + requests: {type: integer, minimum: 0} + tokens: {type: integer, minimum: 0} + error_code: {type: string} + last_error_code: {type: string, nullable: true} + blocked_at: {type: string, format: date-time} + retried_at: {type: string, nullable: true, format: date-time} + retry_request_generation: {type: integer, nullable: true, minimum: 1} + TopicMemoryProcessingState: + type: string + enum: [idle, pending, blocked] + TopicMemoryProcessingRetryStatus: + type: string + enum: [accepted] + TopicMemoryProcessingStatus: + type: object + additionalProperties: false + required: [state, source_cursor, source_head, target_source_through, block] + properties: + state: {$ref: "#/components/schemas/TopicMemoryProcessingState"} + source_cursor: {type: integer, minimum: 0} + source_head: {type: integer, minimum: 0} + target_source_through: {type: integer, nullable: true, minimum: 0} + block: + nullable: true + allOf: + - {$ref: "#/components/schemas/TopicMemoryProcessingBlock"} + RetryTopicMemoryProcessingResponse: + type: object + additionalProperties: false + required: [status, block] + properties: + status: {$ref: "#/components/schemas/TopicMemoryProcessingRetryStatus"} + block: {$ref: "#/components/schemas/TopicMemoryProcessingBlock"} FlushTopicMemoryRequest: type: object additionalProperties: false diff --git a/src/powercontext/builtin/artifacts/topic_memory/processing.py b/src/powercontext/builtin/artifacts/topic_memory/processing.py new file mode 100644 index 0000000000..c344e7f0de --- /dev/null +++ b/src/powercontext/builtin/artifacts/topic_memory/processing.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Operational Topic processing state, containing no Source or model content.""" + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel + +from powercontext.errors import PowerContextError + + +class TopicMemoryWorkBlock(BaseModel): + block_id: str + attempt_id: str + scope_id: str + binding_name: str + source_after: int + source_through: int + target_source_through: int + attempts: int + requests: int + tokens: int + error_code: str + last_error_code: str | None + blocked_at: datetime + retried_at: datetime | None = None + retry_request_generation: int | None = None + + +class TopicMemoryProcessingStatus(BaseModel): + state: Literal["idle", "pending", "blocked"] + source_cursor: int + source_head: int + target_source_through: int | None + block: TopicMemoryWorkBlock | None = None + + +class TopicMemoryRetryResult(BaseModel): + status: Literal["accepted"] = "accepted" + block: TopicMemoryWorkBlock + + +class TopicMemoryBlockNotFoundError(PowerContextError, RuntimeError): + """The requested block does not belong to this Scope.""" + + +class TopicMemoryBlockConflictError(PowerContextError, RuntimeError): + """The requested block no longer owns the current frontier.""" diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index c3edefbe07..aedcf50f4e 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -628,6 +628,39 @@ def _entry_text_type(): ) +TOPIC_MEMORY_WORK_ATTEMPTS_TABLE = Table( + "pc_topic_memory_work_attempts", + SHARED_METADATA, + Column("block_id", identity_string(36), primary_key=True), + Column("attempt_id", identity_string(36), nullable=False), + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), nullable=False), + Column("binding_name", identity_string(MAX_BINDING_NAME_LENGTH), nullable=False), + Column("source_after", BigInteger, nullable=False), + Column("source_through", BigInteger, nullable=False), + Column("target_source_through", BigInteger, nullable=False), + Column("attempts", Integer, nullable=False), + Column("requests", BigInteger, nullable=False), + Column("tokens", BigInteger, nullable=False), + Column("last_error_code", String(64), nullable=True), + Column("blocked_code", String(64), nullable=True), + Column("started_at", DateTime(timezone=False), nullable=True), + Column("finished_at", DateTime(timezone=False), nullable=True), + Column("blocked_at", DateTime(timezone=False), nullable=True), + Column("retried_at", DateTime(timezone=False), nullable=True), + Column("retry_request_generation", BigInteger, nullable=True), + Column("supervisor_group", identity_string(64), nullable=True), + Column("supervisor_generation", BigInteger, nullable=True), + Column("holder_id", identity_string(36), nullable=True), + CheckConstraint( + "source_after >= 0 AND source_through > source_after AND target_source_through >= source_through", + name="ck_pc_topic_attempt_window", + ), + CheckConstraint("attempts > 0 AND requests >= 0 AND tokens >= 0", name="ck_pc_topic_attempt_usage"), + UniqueConstraint("scope_id", "binding_name", "source_after", "attempt_id", name="uq_pc_topic_attempt_identity"), + Index("ix_pc_topic_attempt_frontier", "scope_id", "binding_name", "source_after", "blocked_at"), +) + + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE = Table( "pc_topic_memory_revision_publications", SHARED_METADATA, @@ -987,6 +1020,7 @@ def _entry_text_type(): ARTIFACT_PROCESSING_LEASES_TABLE, ARTIFACT_PROCESSING_BINDING_STATES_TABLE, TOPIC_MEMORY_WORK_BUDGETS_TABLE, + TOPIC_MEMORY_WORK_ATTEMPTS_TABLE, ARTIFACT_PROCESSING_PENDING_TABLE, ARTIFACT_PROCESSING_AUTO_WAVE_TARGETS_TABLE, ARTIFACT_PROCESSING_SEQUENCES_TABLE, diff --git a/src/powercontext/builtin/persistence/topic_memory_budget.py b/src/powercontext/builtin/persistence/topic_memory_budget.py index 7c9b45ac0b..9f203dfdcc 100644 --- a/src/powercontext/builtin/persistence/topic_memory_budget.py +++ b/src/powercontext/builtin/persistence/topic_memory_budget.py @@ -27,8 +27,17 @@ from powercontext.builtin.persistence.cursors import SourceCursorRepository from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import GenerationConflictError -from powercontext.builtin.persistence.supervision import ArtifactProcessingFence, ArtifactProcessingLeaseRepository -from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, TOPIC_MEMORY_WORK_BUDGETS_TABLE +from powercontext.builtin.persistence.supervision import ( + ArtifactProcessingFence, + ArtifactProcessingLeaseRepository, + database_utc_now, +) +from powercontext.builtin.persistence.tables import ( + SOURCE_JOURNAL_HEADS_TABLE, + TOPIC_MEMORY_PROCESSING_TARGETS_TABLE, + TOPIC_MEMORY_WORK_ATTEMPTS_TABLE, + TOPIC_MEMORY_WORK_BUDGETS_TABLE, +) # Server-owned ceilings, shared by ALL attempts and stages at the same Cursor. MAX_TOPIC_MEMORY_WORK_ATTEMPTS = 3 @@ -64,23 +73,18 @@ def exhausted_reason(row: Mapping[Any, Any]) -> str: async def load_topic_memory_work_budget( - connection: AsyncConnection, scope_id: str, binding_name: str, source_after: int + connection: AsyncConnection, scope_id: str, binding_name: str, source_after: int, *, for_update: bool = False ) -> Mapping[Any, Any] | None: """Read one authoritative frontier's metadata without changing its allowance.""" table = TOPIC_MEMORY_WORK_BUDGETS_TABLE - return ( - ( - await connection.execute( - select(table).where( - table.c.scope_id == scope_id, - table.c.binding_name == binding_name, - table.c.source_after == source_after, - ) - ) - ) - .mappings() - .one_or_none() + statement = select(table).where( + table.c.scope_id == scope_id, + table.c.binding_name == binding_name, + table.c.source_after == source_after, ) + if for_update: + statement = statement.with_for_update() + return (await connection.execute(statement)).mappings().one_or_none() async def require_topic_memory_work_available( @@ -92,6 +96,82 @@ async def require_topic_memory_work_available( raise TopicMemoryGenerationError(reason) +async def load_or_create_topic_memory_work_attempt(connection: AsyncConnection, row: Mapping[Any, Any]): + """Retain an exact attempt snapshot under the caller's Source-head lock. + + Legacy rows have no trustworthy start time, owner or last error. Observation + must not invent these historical facts. + """ + table = TOPIC_MEMORY_WORK_ATTEMPTS_TABLE + key = ( + table.c.attempt_id == row["attempt_id"], + table.c.scope_id == row["scope_id"], + table.c.binding_name == row["binding_name"], + table.c.source_after == row["source_after"], + ) + existing = (await connection.execute(select(table).where(*key).with_for_update())).mappings().one_or_none() + if existing is not None: + return existing + target = await connection.scalar( + select(TOPIC_MEMORY_PROCESSING_TARGETS_TABLE.c.source_through) + .where( + TOPIC_MEMORY_PROCESSING_TARGETS_TABLE.c.scope_id == row["scope_id"], + TOPIC_MEMORY_PROCESSING_TARGETS_TABLE.c.binding_name == row["binding_name"], + ) + .with_for_update() + ) + await connection.execute( + insert(table).values( + block_id=str(uuid4()), + **{ + name: row[name] + for name in ( + "attempt_id", + "scope_id", + "binding_name", + "source_after", + "source_through", + "attempts", + "requests", + "tokens", + ) + }, + target_source_through=max(row["source_through"], target or 0), + last_error_code=str(row["failure_code"]) or None, + ) + ) + return (await connection.execute(select(table).where(*key))).mappings().one() + + +async def record_topic_memory_work_failure( + connection: AsyncConnection, scope_id: str, binding_name: str, error_code: str, *, attempt_id: str | None = None +) -> None: + """Settle a failed attempt after Worker exit without changing its allowance.""" + await connection.execute( + update(SOURCE_JOURNAL_HEADS_TABLE) + .where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope_id) + .values(position=SOURCE_JOURNAL_HEADS_TABLE.c.position) + ) + cursor = await SourceCursorRepository().load(connection, scope_id, binding_name, for_update=True) + row = await load_topic_memory_work_budget( + connection, scope_id, binding_name, 0 if cursor is None else cursor.cursor.sequence, for_update=True + ) + if row is None or (attempt_id is not None and row["attempt_id"] != attempt_id): + return + attempt = await load_or_create_topic_memory_work_attempt(connection, row) + table = TOPIC_MEMORY_WORK_ATTEMPTS_TABLE + await connection.execute( + update(table) + .where(table.c.block_id == attempt["block_id"]) + .values( + finished_at=attempt["finished_at"] or await database_utc_now(connection), + last_error_code=attempt["last_error_code"] or error_code, + requests=row["requests"], + tokens=row["tokens"], + ) + ) + + class TopicMemoryWorkBudget: """Commit worst-case charges before I/O; never refund ambiguous/failed work. @@ -110,6 +190,7 @@ def __init__( source_through: int, cursor_generation: int | None, fence: ArtifactProcessingFence, + target_source_through: int | None = None, ) -> None: self.database = database self.scope_id = scope_id @@ -118,6 +199,7 @@ def __init__( self.source_through = source_through self.cursor_generation = cursor_generation self.fence = fence + self.target_source_through = source_through if target_source_through is None else target_source_through self.attempt_id = str(uuid4()) def _key(self): @@ -178,6 +260,26 @@ async def begin(self) -> None: source_through=max(self.source_through, row["source_through"]), ) ) + if not reason: + current = await self._current(connection) + await connection.execute( + insert(TOPIC_MEMORY_WORK_ATTEMPTS_TABLE).values( + block_id=str(uuid4()), + attempt_id=self.attempt_id, + scope_id=self.scope_id, + binding_name=self.binding_name, + source_after=self.source_after, + source_through=current["source_through"], + target_source_through=max(self.target_source_through, current["source_through"]), + attempts=current["attempts"], + requests=current["requests"], + tokens=current["tokens"], + started_at=await database_utc_now(connection), + supervisor_group=self.fence.supervisor_group, + supervisor_generation=self.fence.supervisor_generation, + holder_id=self.fence.holder_id, + ) + ) if reason: raise TopicMemoryGenerationError(reason) @@ -232,6 +334,17 @@ async def fail(self, reason: str) -> None: failure_code=reason, ) ) + await record_topic_memory_work_failure( + connection, self.scope_id, self.binding_name, reason, attempt_id=self.attempt_id + ) + + async def record_failure(self, code: str) -> None: + """Keep a last-error category separately from deterministic rejection policy.""" + async with self.database.transaction() as connection: + await self._lock(connection) + await record_topic_memory_work_failure( + connection, self.scope_id, self.binding_name, code, attempt_id=self.attempt_id + ) async def complete(self, connection: AsyncConnection) -> None: """Called only inside the atomic publisher, before its Cursor CAS.""" @@ -239,4 +352,18 @@ async def complete(self, connection: AsyncConnection) -> None: code = str(row["failure_code"]) if code and code not in RECOVERABLE_TOPIC_MEMORY_REJECTIONS: raise TopicMemoryGenerationError(code) + await connection.execute( + update(TOPIC_MEMORY_WORK_ATTEMPTS_TABLE) + .where( + TOPIC_MEMORY_WORK_ATTEMPTS_TABLE.c.attempt_id == self.attempt_id, + TOPIC_MEMORY_WORK_ATTEMPTS_TABLE.c.scope_id == self.scope_id, + TOPIC_MEMORY_WORK_ATTEMPTS_TABLE.c.binding_name == self.binding_name, + TOPIC_MEMORY_WORK_ATTEMPTS_TABLE.c.source_after == self.source_after, + ) + .values( + finished_at=await database_utc_now(connection), + requests=row["requests"], + tokens=row["tokens"], + ) + ) await connection.execute(delete(TOPIC_MEMORY_WORK_BUDGETS_TABLE).where(*self._key())) diff --git a/src/powercontext/builtin/persistence/topic_memory_recovery.py b/src/powercontext/builtin/persistence/topic_memory_recovery.py new file mode 100644 index 0000000000..13a4a4c85f --- /dev/null +++ b/src/powercontext/builtin/persistence/topic_memory_recovery.py @@ -0,0 +1,224 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Durable block observation and conditional, idempotent Topic recovery.""" + +from datetime import UTC + +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.artifacts.topic_memory.processing import ( + TopicMemoryBlockConflictError, + TopicMemoryBlockNotFoundError, + TopicMemoryWorkBlock, +) +from powercontext.builtin.persistence.cursors import SourceCursorRepository +from powercontext.builtin.persistence.processing import ArtifactProcessingPendingRepository +from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository, database_utc_now +from powercontext.builtin.persistence.tables import ( + SOURCE_JOURNAL_HEADS_TABLE as HEADS, +) +from powercontext.builtin.persistence.tables import ( + TOPIC_MEMORY_PROCESSING_TARGETS_TABLE as TARGETS, +) +from powercontext.builtin.persistence.tables import ( + TOPIC_MEMORY_WORK_ATTEMPTS_TABLE as ATTEMPTS, +) +from powercontext.builtin.persistence.tables import ( + TOPIC_MEMORY_WORK_BUDGETS_TABLE as BUDGETS, +) +from powercontext.builtin.persistence.topic_memory_budget import ( + exhausted_reason, + load_or_create_topic_memory_work_attempt, + load_topic_memory_work_budget, +) + + +async def observe_topic_memory_block( + connection: AsyncConnection, scope_id: str, binding_name: str, *, worker_stopped: bool = False +) -> TopicMemoryWorkBlock | None: + """Record the first observation; an active final attempt may still publish. + + Only the dispatcher/failed Worker can attest that the Worker has stopped. + Status readers instead require a finished attempt or an obsolete lease. + Legacy rows carry an observation timestamp, never an invented failure time. + """ + cursor = await SourceCursorRepository().load(connection, scope_id, binding_name) + after = 0 if cursor is None else cursor.cursor.sequence + row = await load_topic_memory_work_budget(connection, scope_id, binding_name, after) + if row is None or not exhausted_reason(row): + return None + observed_attempt = None + lease = None + if not worker_stopped: + observed_attempt = ( + ( + await connection.execute( + select(ATTEMPTS).where( + ATTEMPTS.c.scope_id == scope_id, + ATTEMPTS.c.binding_name == binding_name, + ATTEMPTS.c.source_after == after, + ATTEMPTS.c.attempt_id == row["attempt_id"], + ) + ) + ) + .mappings() + .one_or_none() + ) + if ( + observed_attempt is not None + and observed_attempt["started_at"] is not None + and observed_attempt["finished_at"] is None + ): + # A current locking read avoids a stale expired lease in MySQL's + # repeatable-read snapshot. Match publication's lease -> head order. + lease = await ArtifactProcessingLeaseRepository().load( + connection, observed_attempt["supervisor_group"], for_update=True + ) + # Serialize observation with capture, begin, publication and explicit retry. + await connection.execute(update(HEADS).where(HEADS.c.scope_id == scope_id).values(position=HEADS.c.position)) + cursor = await SourceCursorRepository().load(connection, scope_id, binding_name, for_update=True) + after = 0 if cursor is None else cursor.cursor.sequence + row = await load_topic_memory_work_budget(connection, scope_id, binding_name, after, for_update=True) + if row is None or not (reason := exhausted_reason(row)): + return None + target = await connection.scalar( + select(TARGETS.c.source_through) + .where(TARGETS.c.scope_id == scope_id, TARGETS.c.binding_name == binding_name) + .with_for_update() + ) + head = await connection.scalar(select(HEADS.c.position).where(HEADS.c.scope_id == scope_id).with_for_update()) + through = head if target is None else target + if through is None or through <= after: + return None + attempt = await load_or_create_topic_memory_work_attempt(connection, row) + if attempt["blocked_at"] is not None: + return _block(attempt) + if not worker_stopped and attempt["started_at"] is not None and attempt["finished_at"] is None: + if observed_attempt is None or observed_attempt["block_id"] != attempt["block_id"]: + # A replacement attempt appeared while acquiring the locks. Its + # owner was not inspected, so it must remain pending on this read. + return None + now = await database_utc_now(connection) + if ( + lease is not None + and lease.holder_id == attempt["holder_id"] + and lease.supervisor_generation == attempt["supervisor_generation"] + and (lease.lease_expires_at is None or lease.lease_expires_at > now) + ): + return None + now = await database_utc_now(connection) + await connection.execute( + update(ATTEMPTS) + .where(ATTEMPTS.c.block_id == attempt["block_id"]) + .values( + blocked_code=reason, + blocked_at=now, + finished_at=attempt["finished_at"] or now, + requests=row["requests"], + tokens=row["tokens"], + ) + ) + return await get_topic_memory_block(connection, scope_id, binding_name, attempt["block_id"]) + + +async def get_topic_memory_block( + connection: AsyncConnection, scope_id: str, binding_name: str, block_id: str +) -> TopicMemoryWorkBlock: + row = ( + ( + await connection.execute( + select(ATTEMPTS).where( + ATTEMPTS.c.block_id == block_id, + ATTEMPTS.c.scope_id == scope_id, + ATTEMPTS.c.binding_name == binding_name, + ATTEMPTS.c.blocked_at.is_not(None), + ) + ) + ) + .mappings() + .one_or_none() + ) + if row is None: + raise TopicMemoryBlockNotFoundError + return _block(row) + + +async def retry_topic_memory_block( + connection: AsyncConnection, scope_id: str, binding_name: str, block_id: str +) -> TopicMemoryWorkBlock: + """Grant one new allowance for an observed block, retaining its audit record.""" + await connection.execute(update(HEADS).where(HEADS.c.scope_id == scope_id).values(position=HEADS.c.position)) + block = await get_topic_memory_block(connection, scope_id, binding_name, block_id) + if block.retried_at is not None: + # A lost HTTP response must never reset a later attempt at the same Cursor. + return block + cursor = await SourceCursorRepository().load(connection, scope_id, binding_name, for_update=True) + after = 0 if cursor is None else cursor.cursor.sequence + row = await load_topic_memory_work_budget(connection, scope_id, binding_name, after, for_update=True) + if ( + row is None + or after != block.source_after + or row["attempt_id"] != block.attempt_id + or exhausted_reason(row) != block.error_code + ): + raise TopicMemoryBlockConflictError + await connection.execute( + delete(BUDGETS).where( + BUDGETS.c.scope_id == scope_id, + BUDGETS.c.binding_name == binding_name, + BUDGETS.c.source_after == after, + BUDGETS.c.attempt_id == block.attempt_id, + ) + ) + pending = await ArtifactProcessingPendingRepository().request_flush(connection, scope_id, binding_name) + if pending is None: + raise TopicMemoryBlockConflictError + intent = await ArtifactProcessingIntentRepository().request(connection, scope_id, binding_name) + await connection.execute( + update(ATTEMPTS) + .where(ATTEMPTS.c.block_id == block_id) + .values( + retried_at=await database_utc_now(connection), + retry_request_generation=intent.requested_generation, + ) + ) + return await get_topic_memory_block(connection, scope_id, binding_name, block_id) + + +def _block(row) -> TopicMemoryWorkBlock: + return TopicMemoryWorkBlock( + block_id=row["block_id"], + **{ + name: row[name] + for name in ( + "attempt_id", + "scope_id", + "binding_name", + "source_after", + "source_through", + "target_source_through", + "attempts", + "requests", + "tokens", + "last_error_code", + "retry_request_generation", + ) + }, + error_code=row["blocked_code"], + blocked_at=row["blocked_at"].replace(tzinfo=UTC), + retried_at=None if row["retried_at"] is None else row["retried_at"].replace(tzinfo=UTC), + ) diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 75650cfd20..8468329864 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -123,6 +123,11 @@ TopicMemorySearchMode, TopicMemorySearchResult, ) +from powercontext.builtin.artifacts.topic_memory.processing import ( + TopicMemoryProcessingStatus, + TopicMemoryRetryResult, + TopicMemoryWorkBlock, +) from powercontext.builtin.code.application import CodeApplication from powercontext.builtin.code.errors import CodeError from powercontext.builtin.code.models import CodeConfig, CodeQueryRequest, CodeQueryResult @@ -261,6 +266,7 @@ ) from powercontext.builtin.runtime.skill_search import search_skill_library from powercontext.builtin.runtime.statistics import RelationalScopedStatistics, overview_selection +from powercontext.builtin.runtime.topic_memory_recovery import TopicMemoryRecoveryService from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher from powercontext.builtin.scope import ScopeApplication, ScopeDescriptor, ScopeSelection from powercontext.builtin.scope.subject_sources import SubjectSourceService @@ -3228,6 +3234,45 @@ async def browse( async with self._runtime._scoped_operation(self.scope_id): return await self._runtime._topic_memory_browse(self.scope_id, limit=limit, after=after) + async def processing_status(self) -> TopicMemoryProcessingStatus: + if self._runtime._topic_memory_recovery is None: + raise TopicMemoryProcessingUnavailableError + async with self._runtime._scoped_operation(self.scope_id): + return await self._runtime._topic_memory_recovery.status(self.scope_id) + + async def get_block(self, block_id: str) -> TopicMemoryWorkBlock: + if self._runtime._topic_memory_recovery is None: + raise TopicMemoryProcessingUnavailableError + async with self._runtime._scoped_operation(self.scope_id): + return await self._runtime._topic_memory_recovery.get_block(self.scope_id, block_id) + + async def retry(self, block_id: str) -> TopicMemoryRetryResult: + if self._runtime._topic_memory_recovery is None: + raise TopicMemoryProcessingUnavailableError + async with self._runtime._scoped_operation(self.scope_id), self._runtime._locked(self.scope_id): + if not self._runtime._topic_memory_processing_available: + block = await self._runtime._topic_memory_recovery.get_block(self.scope_id, block_id) + if block.retried_at is None: + raise TopicMemoryProcessingUnavailableError + # Reading an old acceptance does not require a configured Worker. + return TopicMemoryRetryResult(block=block) + result = await self._runtime._topic_memory_recovery.retry(self.scope_id, block_id) + if self._runtime.artifact_processing_supervisor is not None: + try: + self._runtime.artifact_processing_supervisor.wake(TOPIC_MEMORY_SOURCE_WINDOW_BINDING) + except Exception: + log_safely( + logger, + logging.WARNING, + "Topic Memory supervisor wake failed after retry acceptance", + extra={ + "event": "topic_memory.retry.wake_failed", + "outcome": "accepted", + "unit": "topic-memory", + }, + ) + return result + async def flush(self) -> TopicMemoryFlushResult: if not self._runtime._topic_memory_processing_available or self._runtime._topic_memory_flush is None: raise TopicMemoryProcessingUnavailableError @@ -3454,6 +3499,7 @@ def __init__( topic_memory_get: TopicMemoryGet | None = None, topic_memory_browse: TopicMemoryBrowse | None = None, topic_memory_flush: TopicMemoryFlush | None = None, + topic_memory_recovery: TopicMemoryRecoveryService | None = None, topic_memory_embedding_model: EmbeddingModel | None = None, topic_memory_processing_available: bool = False, topic_memory_search_observer: TopicMemorySearchObserver | None = None, @@ -3517,6 +3563,7 @@ def __init__( self._topic_memory_get = topic_memory_get self._topic_memory_browse = topic_memory_browse self._topic_memory_flush = topic_memory_flush + self._topic_memory_recovery = topic_memory_recovery self._topic_memory_embedding_model = topic_memory_embedding_model self._topic_memory_processing_available = topic_memory_processing_available self._topic_memory_search_observer = topic_memory_search_observer diff --git a/src/powercontext/builtin/runtime/artifact_processing.py b/src/powercontext/builtin/runtime/artifact_processing.py index edfe93a419..660cb5d104 100644 --- a/src/powercontext/builtin/runtime/artifact_processing.py +++ b/src/powercontext/builtin/runtime/artifact_processing.py @@ -125,6 +125,7 @@ class ArtifactProcessingBinding: # Applies to every invocation, including explicit requests and retries. # Domains inspect durable metadata here; they retain their Worker guards. work_block: Callable[[AsyncConnection, str, str], Awaitable[ArtifactProcessingBlock | None]] | None = None + work_failure: Callable[[AsyncConnection, str, str, str], Awaitable[None]] | None = None def __post_init__(self) -> None: if not self.binding_name or self.binding_name != self.binding_name.strip(): @@ -1178,6 +1179,34 @@ async def _reap(self, state: _FamilyState) -> None: # noqa: C901 - distinct Wor if block is not None: self._wait_for_block(state, running.assignment, block) continue + if state.binding.work_failure is not None: + code = ( + processing_error_code(error.failure.exception_type, error.failure.error_code) + if isinstance(error, _WorkerExecutionError) + else processing_error_code(type(error), _safe_error_attribute(error, "error_code", "")) + ) + try: + async with ( + asyncio.timeout(_ADMISSION_PROBE_TIMEOUT_SECONDS), + self._database.transaction() as connection, + ): + await self._leases.require_fence(connection, running.assignment.fence) + await state.binding.work_failure(connection, scope, state.binding.binding_name, code) + except ArtifactProcessingLeadershipLostError: + raise + except Exception as recording_error: + log_safely( + logger, + logging.WARNING, + "Failed to retain processing failure diagnostics", + extra={ + "event": "artifact_processing.failure_record_failed", + "family": state.binding.artifact_family, + "binding": state.binding.binding_name, + "scope": scope, + "exception_type": type(recording_error).__name__, + }, + ) previous = state.retries.get(scope) failures = 1 if previous is None else previous.failures + 1 delay = self._retry_delay(failures) diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index b2c0cd1b99..54d1627f3f 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -126,6 +126,7 @@ from powercontext.builtin.persistence.tables import BUILTIN_TABLES from powercontext.builtin.persistence.tag_schema import ensure_topic_memory_tag_schema from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository +from powercontext.builtin.persistence.topic_memory_budget import record_topic_memory_work_failure from powercontext.builtin.persistence.topic_memory_index import ( CompositeTopicMemoryIndex, TopicMemoryIndex, @@ -189,6 +190,7 @@ run_topic_memory_worker, validate_topic_memory_provider_settings, ) +from powercontext.builtin.runtime.topic_memory_recovery import TopicMemoryRecoveryService from powercontext.builtin.runtime.topic_memory_scope import topic_memory_processing_block from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher from powercontext.builtin.sources import ( @@ -740,6 +742,7 @@ async def run_profile(scope_id, high): topic_memory_get=contexts.get_topic_memory, topic_memory_browse=contexts.browse_topic_memories, topic_memory_flush=contexts.request_topic_memory_flush, + topic_memory_recovery=TopicMemoryRecoveryService(contexts.database), topic_memory_embedding_model=configured_embedding, topic_memory_processing_available=topic_memory_processing_available, topic_memory_search_observer=topic_memory_search_observer, @@ -900,6 +903,7 @@ def _artifact_processing_bindings( # noqa: C901 - validate and assemble one reg else SourceProcessingPendingProvider(contexts.database, binding, family), automatic_scope_filter=enabled_profile_scopes if family == "profile" else None, work_block=topic_memory_processing_block if family == "topic-memory" else None, + work_failure=record_topic_memory_work_failure if family == "topic-memory" else None, ) ) _validate_processing_registrations(configured) diff --git a/src/powercontext/builtin/runtime/topic_memory_processing.py b/src/powercontext/builtin/runtime/topic_memory_processing.py index 9d50c7d14b..6530b228d5 100644 --- a/src/powercontext/builtin/runtime/topic_memory_processing.py +++ b/src/powercontext/builtin/runtime/topic_memory_processing.py @@ -518,6 +518,7 @@ async def process( source_through=assignment.source_through, cursor_generation=assignment.cursor_generation, fence=assignment.fence, + target_source_through=assignment.wave_target, ) token = self._work_budget.set(budget) self._used_stages.clear() @@ -535,6 +536,8 @@ async def process( except TopicMemoryGenerationError as error: if error.code in RECORDED_TOPIC_MEMORY_REJECTIONS: await budget.fail(error.code) + else: + await budget.record_failure(error.code) raise except ArtifactProcessingLeadershipLostError: return ArtifactProcessingWorkerCompletion(ArtifactProcessingWorkerOutcome.LEADERSHIP_LOST) diff --git a/src/powercontext/builtin/runtime/topic_memory_recovery.py b/src/powercontext/builtin/runtime/topic_memory_recovery.py new file mode 100644 index 0000000000..831c7829a1 --- /dev/null +++ b/src/powercontext/builtin/runtime/topic_memory_recovery.py @@ -0,0 +1,84 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Scope-bound inspection and explicit retry of Topic processing frontiers.""" + +from sqlalchemy import select, update + +from powercontext.builtin.artifacts.topic_memory import TOPIC_MEMORY_SOURCE_WINDOW_BINDING +from powercontext.builtin.artifacts.topic_memory.processing import ( + TopicMemoryProcessingStatus, + TopicMemoryRetryResult, + TopicMemoryWorkBlock, +) +from powercontext.builtin.persistence.cursors import SourceCursorRepository +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, TOPIC_MEMORY_PROCESSING_TARGETS_TABLE +from powercontext.builtin.persistence.topic_memory_recovery import ( + get_topic_memory_block, + observe_topic_memory_block, + retry_topic_memory_block, +) + + +class TopicMemoryRecoveryService: + def __init__(self, database: AsyncDatabase) -> None: + self._database = database + + async def status(self, scope_id: str) -> TopicMemoryProcessingStatus: + async with self._database.transaction() as connection: + block = await observe_topic_memory_block(connection, scope_id, TOPIC_MEMORY_SOURCE_WINDOW_BINDING) + # Current reads under the common head lock keep the response aligned + # with the observed block, even when an earlier snapshot was pinned. + await connection.execute( + update(SOURCE_JOURNAL_HEADS_TABLE) + .where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope_id) + .values(position=SOURCE_JOURNAL_HEADS_TABLE.c.position) + ) + cursor = await SourceCursorRepository().load( + connection, scope_id, TOPIC_MEMORY_SOURCE_WINDOW_BINDING, for_update=True + ) + after = 0 if cursor is None else cursor.cursor.sequence + head = ( + await connection.scalar( + select(SOURCE_JOURNAL_HEADS_TABLE.c.position) + .where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope_id) + .with_for_update() + ) + or 0 + ) + target = await connection.scalar( + select(TOPIC_MEMORY_PROCESSING_TARGETS_TABLE.c.source_through) + .where( + TOPIC_MEMORY_PROCESSING_TARGETS_TABLE.c.scope_id == scope_id, + TOPIC_MEMORY_PROCESSING_TARGETS_TABLE.c.binding_name == TOPIC_MEMORY_SOURCE_WINDOW_BINDING, + ) + .with_for_update() + ) + return TopicMemoryProcessingStatus( + state="blocked" if block is not None else "idle" if after >= head and target is None else "pending", + source_cursor=after, + source_head=head, + target_source_through=target, + block=block, + ) + + async def get_block(self, scope_id: str, block_id: str) -> TopicMemoryWorkBlock: + async with self._database.transaction() as connection: + return await get_topic_memory_block(connection, scope_id, TOPIC_MEMORY_SOURCE_WINDOW_BINDING, block_id) + + async def retry(self, scope_id: str, block_id: str) -> TopicMemoryRetryResult: + async with self._database.transaction() as connection: + block = await retry_topic_memory_block(connection, scope_id, TOPIC_MEMORY_SOURCE_WINDOW_BINDING, block_id) + return TopicMemoryRetryResult(block=block) diff --git a/src/powercontext/builtin/runtime/topic_memory_scope.py b/src/powercontext/builtin/runtime/topic_memory_scope.py index 901a8ebaf4..a86e318c57 100644 --- a/src/powercontext/builtin/runtime/topic_memory_scope.py +++ b/src/powercontext/builtin/runtime/topic_memory_scope.py @@ -30,7 +30,7 @@ from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, TOPIC_MEMORY_PROCESSING_TARGETS_TABLE -from powercontext.builtin.persistence.topic_memory_budget import exhausted_reason, load_topic_memory_work_budget +from powercontext.builtin.persistence.topic_memory_recovery import observe_topic_memory_block from powercontext.builtin.runtime.processing_contracts import ( ArtifactProcessingBlock, ArtifactProcessingBlockedError, @@ -66,28 +66,16 @@ async def topic_memory_processing_block( Input selection, projections, token estimation and models stay in the Worker. A finished target must still be allowed to retire its intent. """ - cursor = await SourceCursorRepository().load(connection, scope_id, binding_name) - source_after = 0 if cursor is None else cursor.cursor.sequence - targets = TOPIC_MEMORY_PROCESSING_TARGETS_TABLE - target = await connection.scalar( - select(targets.c.source_through).where(targets.c.scope_id == scope_id, targets.c.binding_name == binding_name) - ) - through = target - if through is None: - through = await connection.scalar( - select(SOURCE_JOURNAL_HEADS_TABLE.c.position).where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope_id) - ) - if through is None or through <= source_after: - return None - row = await load_topic_memory_work_budget(connection, scope_id, binding_name, source_after) - if row is None or not (reason := exhausted_reason(row)): + block = await observe_topic_memory_block(connection, scope_id, binding_name, worker_stopped=True) + if block is None: return None return ArtifactProcessingBlock( - key=str(source_after), + key=block.block_id, stage="topic_memory", - error_code=reason, + error_code=block.error_code, details=tuple( - (name, int(row[name])) for name in ("source_after", "source_through", "attempts", "requests", "tokens") + (name, int(getattr(block, name))) + for name in ("source_after", "source_through", "attempts", "requests", "tokens") ), ) @@ -150,13 +138,22 @@ async def process(self, assignment: ArtifactProcessingWorkAssignment, /) -> Arti except GenerationConflictError: return ArtifactProcessingWorkerCompletion(ArtifactProcessingWorkerOutcome.CURSOR_CONFLICT) except TopicMemoryGenerationError as error: - async with self._database.transaction() as connection: - block = await topic_memory_processing_block(connection, assignment.scope_id, assignment.binding_name) - if block is not None and block.error_code == error.code: - raise ArtifactProcessingBlockedError(block) from error - raise + return await self._generation_failure(assignment, error) raise TopicMemoryGenerationError("target_did_not_converge") + async def _generation_failure( + self, assignment: ArtifactProcessingWorkAssignment, error: TopicMemoryGenerationError + ) -> ArtifactProcessingWorkerCompletion: + async with self._database.transaction() as connection: + try: + await self._leases.require_fence(connection, assignment.fence) + except ArtifactProcessingLeadershipLostError: + return ArtifactProcessingWorkerCompletion(ArtifactProcessingWorkerOutcome.LEADERSHIP_LOST) + block = await topic_memory_processing_block(connection, assignment.scope_id, assignment.binding_name) + if block is not None and block.error_code == error.code: + raise ArtifactProcessingBlockedError(block) from error + raise error + async def _target(self, assignment: ArtifactProcessingWorkAssignment) -> TopicMemoryProcessingTarget | None: async with self._database.transaction() as connection: await self._leases.require_fence(connection, assignment.fence) diff --git a/src/powercontext/client/cli.py b/src/powercontext/client/cli.py index a421a2bbc6..f5ef8adbae 100644 --- a/src/powercontext/client/cli.py +++ b/src/powercontext/client/cli.py @@ -132,6 +132,14 @@ SubtreeScopeSelection, UnpublishRemoteSkillRequest, ) +from powercontext.http._generated.models import ( + GetTopicMemoryProcessingBlockRequest, + RetryTopicMemoryProcessingRequest, + RetryTopicMemoryProcessingResponse, + TopicMemoryProcessingBlock, + TopicMemoryProcessingStatus, + TopicMemoryProcessingStatusRequest, +) HELP_OPTION_NAMES = ("-h", "--help") _ClientResponse: TypeAlias = ( @@ -153,6 +161,9 @@ | ScanExternalSkillsResponse | SkillArtifact | ScopedStats + | TopicMemoryProcessingBlock + | TopicMemoryProcessingStatus + | RetryTopicMemoryProcessingResponse ) _ClientOperation: TypeAlias = Callable[[PowerContextClient], Awaitable[_ClientResponse]] @@ -163,6 +174,59 @@ no_args_is_help=True, ) +topic_memory_app = typer.Typer( + name="topic-memory", + context_settings={"help_option_names": HELP_OPTION_NAMES}, + help="Inspect and explicitly retry blocked Topic Memory processing.", + no_args_is_help=True, +) + + +@topic_memory_app.command("status") +def topic_memory_status( + context: typer.Context, scope_id: Annotated[str, typer.Option(help="Scope to inspect.")] +) -> None: + asyncio.run( + _execute( + context, + lambda client: client.get_topic_memory_processing_status( + TopicMemoryProcessingStatusRequest(scope_id=scope_id) + ), + ) + ) + + +@topic_memory_app.command("block") +def topic_memory_block( + context: typer.Context, + scope_id: Annotated[str, typer.Option(help="Scope containing the block.")], + block_id: Annotated[str, typer.Option(help="Exact block ID returned by status.")], +) -> None: + asyncio.run( + _execute( + context, + lambda client: client.get_topic_memory_processing_block( + GetTopicMemoryProcessingBlockRequest(scope_id=scope_id, block_id=block_id) + ), + ) + ) + + +@topic_memory_app.command("retry") +def topic_memory_retry( + context: typer.Context, + scope_id: Annotated[str, typer.Option(help="Scope to retry; requires scope.admin.")], + block_id: Annotated[str, typer.Option(help="Exact observed block ID; duplicates reuse its acceptance.")], +) -> None: + asyncio.run( + _execute( + context, + lambda client: client.retry_topic_memory_processing( + RetryTopicMemoryProcessingRequest(scope_id=scope_id, block_id=block_id) + ), + ) + ) + @dream_app.command("run") def run_dream( @@ -1737,6 +1801,9 @@ def _print_human_response(response: _ClientResponse) -> None: ListExternalSkillsResponse, ScanExternalSkillsResponse, SkillArtifact, + TopicMemoryProcessingBlock, + TopicMemoryProcessingStatus, + RetryTopicMemoryProcessingResponse, ), ): typer.echo(response.model_dump_json(indent=2)) @@ -1897,6 +1964,7 @@ def register_commands(cli: typer.Typer) -> set[str]: cli.command()(ready) cli.command("import-sessions")(import_sessions) cli.add_typer(dream_app, name="dream") + cli.add_typer(topic_memory_app, name="topic-memory") cli.add_typer(candidate_app, name="candidate") cli.add_typer(experience_app, name="experience") cli.add_typer(skill_app, name="skill") @@ -1908,6 +1976,7 @@ def register_commands(cli: typer.Typer) -> set[str]: "ready", "import-sessions", "dream", + "topic-memory", "candidate", "experience", "skill", diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index e199783e0d..f62b5da047 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -210,8 +210,14 @@ from powercontext.http._generated.models import ( ArtifactTagPage, ArtifactTagSet, + GetTopicMemoryProcessingBlockRequest, QueryArtifactTagsRequest, ReplaceArtifactTagsRequest, + RetryTopicMemoryProcessingRequest, + RetryTopicMemoryProcessingResponse, + TopicMemoryProcessingBlock, + TopicMemoryProcessingStatus, + TopicMemoryProcessingStatusRequest, ) from powercontext.http._generated.operations import ( ACKNOWLEDGE_HANDOFF, @@ -267,6 +273,8 @@ GET_SOURCE, GET_STATS, GET_TOPIC_MEMORY, + GET_TOPIC_MEMORY_PROCESSING_BLOCK, + GET_TOPIC_MEMORY_PROCESSING_STATUS, HANDOFF_CURRENT_WORK, IMPORT_EXTERNAL_SKILL, LIST_ACCESS_AUDIT, @@ -314,6 +322,7 @@ RESOLVE_SCOPE_BINDING, RESOLVE_SCOPE_SELECTION, RETIRE_MEMORY_ENTRY, + RETRY_TOPIC_MEMORY_PROCESSING, REVISE_ARTIFACT_CANDIDATE, REVISE_MEMORY_ENTRY, REVOKE_ACCESS_BINDING, @@ -950,6 +959,24 @@ async def flush_topic_memory(self, request: FlushTopicMemoryRequest) -> FlushTop return await self._request(FLUSH_TOPIC_MEMORY, request) + async def get_topic_memory_processing_status( + self, request: TopicMemoryProcessingStatusRequest + ) -> TopicMemoryProcessingStatus: + """Inspect a frontier without granting another attempt.""" + return await self._request(GET_TOPIC_MEMORY_PROCESSING_STATUS, request) + + async def get_topic_memory_processing_block( + self, request: GetTopicMemoryProcessingBlockRequest + ) -> TopicMemoryProcessingBlock: + """Read the exact retained diagnostics, including a prior accepted retry.""" + return await self._request(GET_TOPIC_MEMORY_PROCESSING_BLOCK, request) + + async def retry_topic_memory_processing( + self, request: RetryTopicMemoryProcessingRequest + ) -> RetryTopicMemoryProcessingResponse: + """Idempotently request one fresh bounded allowance for an observed block.""" + return await self._request(RETRY_TOPIC_MEMORY_PROCESSING, request) + async def get_atomic_memory_state(self, scope_id: str, artifact_id: str) -> atomic_models.AtomicMemoryStateResponse: """Read current state independently of an immutable content revision.""" return await self._request( diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index b0198916e9..4a1f4db5af 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -210,6 +210,7 @@ GetSkillPackageRequest, GetSkillRequest, GetStatsRequest, + GetTopicMemoryProcessingBlockRequest, GetTopicMemoryRequest, HandoffAcknowledgement, HandoffAcknowledgementSelection, @@ -369,6 +370,8 @@ ResolveScopeBindingRequest, ResolveScopeSelectionRequest, RetireMemoryEntryRequest, + RetryTopicMemoryProcessingRequest, + RetryTopicMemoryProcessingResponse, ReviseArtifactCandidateRequest, ReviseMemoryEntryRequest, RevokeAccessBindingRequest, @@ -442,6 +445,11 @@ TopicMemoryArtifact, TopicMemoryFlushStatus, TopicMemoryMatchedBy, + TopicMemoryProcessingBlock, + TopicMemoryProcessingRetryStatus, + TopicMemoryProcessingState, + TopicMemoryProcessingStatus, + TopicMemoryProcessingStatusRequest, TopicMemoryUsedSearchMode, TopicMemoryWriteContent, Trust, @@ -654,6 +662,7 @@ "GetSkillPackageRequest", "GetSkillRequest", "GetStatsRequest", + "GetTopicMemoryProcessingBlockRequest", "GetTopicMemoryRequest", "HandoffAcknowledgement", "HandoffAcknowledgementSelection", @@ -813,6 +822,8 @@ "ResolveScopeSelectionRequest", "ResolvedUsagePeriod", "RetireMemoryEntryRequest", + "RetryTopicMemoryProcessingRequest", + "RetryTopicMemoryProcessingResponse", "ReviseArtifactCandidateRequest", "ReviseMemoryEntryRequest", "RevokeAccessBindingRequest", @@ -886,6 +897,11 @@ "TopicMemoryArtifact", "TopicMemoryFlushStatus", "TopicMemoryMatchedBy", + "TopicMemoryProcessingBlock", + "TopicMemoryProcessingRetryStatus", + "TopicMemoryProcessingState", + "TopicMemoryProcessingStatus", + "TopicMemoryProcessingStatusRequest", "TopicMemoryUsedSearchMode", "TopicMemoryWriteContent", "Trust", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 7111a3050e..9e2b78fd2c 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -1556,6 +1556,79 @@ class FlushMemoryRequest(BaseModel): scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] +class TopicMemoryProcessingStatusRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] + + +class GetTopicMemoryProcessingBlockRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] + block_id: Annotated[StrictStr, Field(max_length=36, min_length=1)] + + +class RetryTopicMemoryProcessingRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] + block_id: Annotated[StrictStr, Field(max_length=36, min_length=1)] + + +class TopicMemoryProcessingBlock(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + block_id: StrictStr + attempt_id: StrictStr + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] + binding_name: StrictStr + source_after: Annotated[StrictInt, Field(ge=0)] + source_through: Annotated[StrictInt, Field(ge=1)] + target_source_through: Annotated[StrictInt, Field(ge=1)] + attempts: Annotated[StrictInt, Field(ge=1)] + requests: Annotated[StrictInt, Field(ge=0)] + tokens: Annotated[StrictInt, Field(ge=0)] + error_code: StrictStr + last_error_code: Annotated[StrictStr | None, Field(...)] + blocked_at: AwareDatetime + retried_at: Annotated[AwareDatetime | None, Field(...)] + retry_request_generation: Annotated[StrictInt | None, Field(ge=1)] + + +class TopicMemoryProcessingState(StrEnum): + IDLE = "idle" + PENDING = "pending" + BLOCKED = "blocked" + + +class TopicMemoryProcessingRetryStatus(StrEnum): + ACCEPTED = "accepted" + + +class TopicMemoryProcessingStatus(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + state: TopicMemoryProcessingState + source_cursor: Annotated[StrictInt, Field(ge=0)] + source_head: Annotated[StrictInt, Field(ge=0)] + target_source_through: Annotated[StrictInt | None, Field(ge=0)] + block: Annotated[TopicMemoryProcessingBlock | None, Field(...)] + + +class RetryTopicMemoryProcessingResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + status: TopicMemoryProcessingRetryStatus + block: TopicMemoryProcessingBlock + + class FlushTopicMemoryRequest(BaseModel): model_config = ConfigDict( extra="forbid", diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index d7aeb4d78d..f19eace9ea 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -88,6 +88,7 @@ GetSkillPackageRequest, GetSkillRequest, GetStatsRequest, + GetTopicMemoryProcessingBlockRequest, GetTopicMemoryRequest, HandoffAcknowledgement, HandoffActivation, @@ -161,6 +162,8 @@ ResolveScopeBindingRequest, ResolveScopeSelectionRequest, RetireMemoryEntryRequest, + RetryTopicMemoryProcessingRequest, + RetryTopicMemoryProcessingResponse, ReviseArtifactCandidateRequest, ReviseMemoryEntryRequest, RevokeAccessBindingRequest, @@ -192,6 +195,9 @@ SourceRecord, SubmitSourceObservationRequest, TopicMemoryArtifact, + TopicMemoryProcessingBlock, + TopicMemoryProcessingStatus, + TopicMemoryProcessingStatusRequest, UnpublishRemoteSkillRequest, UpdateScopeRequest, UpdateSkillLifecycleRequest, @@ -1467,6 +1473,87 @@ class AccessRequirement(BaseModel): access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), ) +GET_TOPIC_MEMORY_PROCESSING_STATUS = Operation[TopicMemoryProcessingStatusRequest, TopicMemoryProcessingStatus]( + method="POST", + path="/v1/topic-memory/processing/status", + operation_id="get_topic_memory_processing_status", + request_type=TopicMemoryProcessingStatusRequest, + request_location="body", + path_parameters=(), + response_type=TopicMemoryProcessingStatus, + success_status=200, + summary="Inspect Topic Memory processing and its current terminal block", + tags=("topic-memory",), + scope_mode="current", + responses={ + 200: { + "description": "Current durable processing state.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + 500: {"$ref": "#/components/responses/InternalError"}, + }, + access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), +) + +GET_TOPIC_MEMORY_PROCESSING_BLOCK = Operation[GetTopicMemoryProcessingBlockRequest, TopicMemoryProcessingBlock]( + method="POST", + path="/v1/topic-memory/processing/blocks/get", + operation_id="get_topic_memory_processing_block", + request_type=GetTopicMemoryProcessingBlockRequest, + request_location="body", + path_parameters=(), + response_type=TopicMemoryProcessingBlock, + success_status=200, + summary="Get a retained Topic Memory processing block", + tags=("topic-memory",), + scope_mode="current", + responses={ + 200: { + "description": "The exact retained block.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + 500: {"$ref": "#/components/responses/InternalError"}, + }, + access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), +) + +RETRY_TOPIC_MEMORY_PROCESSING = Operation[RetryTopicMemoryProcessingRequest, RetryTopicMemoryProcessingResponse]( + method="POST", + path="/v1/topic-memory/processing/retry", + operation_id="retry_topic_memory_processing", + request_type=RetryTopicMemoryProcessingRequest, + request_location="body", + path_parameters=(), + response_type=RetryTopicMemoryProcessingResponse, + success_status=200, + summary="Explicitly retry one observed Topic Memory block", + tags=("topic-memory",), + scope_mode="current", + responses={ + 200: { + "description": "One retry was durably accepted, including duplicate acceptance.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + 500: {"$ref": "#/components/responses/InternalError"}, + }, + access=AccessRequirement(action="scope.admin", resource="scope", scope_id_field="scope_id", resolver="request"), +) + SEARCH_TOPIC_MEMORY = Operation[SearchTopicMemoryRequest, SearchTopicMemoryResponse]( method="POST", path="/v1/topic-memory/search", diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index bf86682eb0..1ed7cb3e9f 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -1884,6 +1884,159 @@ }, } }, + "/v1/topic-memory/processing/status": { + "post": { + "tags": ["topic-memory"], + "summary": "Inspect Topic Memory processing and its current terminal block", + "description": "Return the current " + "Cursor, Source head, " + "retained target and " + "durable blocked-window " + "diagnostics. An active " + "final attempt remains " + "pending because it may " + "still finish. Legacy " + "blocks receive their " + "first observation " + "timestamp; an " + "unavailable historical " + "error code is null. " + "This operation never " + "grants new processing " + "allowance.", + "operationId": "get_topic_memory_processing_status", + "requestBody": { + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/TopicMemoryProcessingStatusRequest"} + } + }, + "required": True, + }, + "responses": { + "200": { + "description": "Current durable processing state.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/TopicMemoryProcessingStatus"}} + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + "500": {"$ref": "#/components/responses/InternalError"}, + }, + "x-powercontext-access": { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "x-powercontext-scope-mode": "current", + } + }, + "/v1/topic-memory/processing/blocks/get": { + "post": { + "tags": ["topic-memory"], + "summary": "Get a retained Topic Memory processing block", + "description": "Return one exact " + "block, including " + "its original " + "diagnostics and any " + "accepted retry, " + "after recovery.", + "operationId": "get_topic_memory_processing_block", + "requestBody": { + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/GetTopicMemoryProcessingBlockRequest"} + } + }, + "required": True, + }, + "responses": { + "200": { + "description": "The exact retained block.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/TopicMemoryProcessingBlock"}} + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + "500": {"$ref": "#/components/responses/InternalError"}, + }, + "x-powercontext-access": { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "x-powercontext-scope-mode": "current", + } + }, + "/v1/topic-memory/processing/retry": { + "post": { + "tags": ["topic-memory"], + "summary": "Explicitly retry one observed Topic Memory block", + "description": "Require scope.admin and " + "the exact block_id " + "returned by status. " + "Atomically retain the " + "block's failure and " + "usage evidence, grant " + "one fresh bounded " + "allowance and request " + "asynchronous processing " + "without advancing the " + "Cursor or discarding the " + "target or Sources. " + "Repeating a block_id " + "returns its original " + "acceptance and cannot " + "reset a later attempt. " + "An unretried block that " + "no longer owns the " + "frontier returns " + "topic_memory_block_conflict. " + "Ordinary flushes, status " + "reads, restarts and " + "model changes never " + "reset allowance.", + "operationId": "retry_topic_memory_processing", + "requestBody": { + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/RetryTopicMemoryProcessingRequest"} + } + }, + "required": True, + }, + "responses": { + "200": { + "description": "One retry was durably accepted, including duplicate acceptance.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/RetryTopicMemoryProcessingResponse"} + } + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + "500": {"$ref": "#/components/responses/InternalError"}, + }, + "x-powercontext-access": { + "action": "scope.admin", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "x-powercontext-scope-mode": "current", + } + }, "/v1/topic-memory/search": { "post": { "tags": ["topic-memory"], @@ -8864,6 +9017,91 @@ "type": "object", "required": ["status", "previous_cursor", "current_cursor", "high_watermark", "processed_source_count"], }, + "TopicMemoryProcessingStatusRequest": { + "properties": {"scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}}, + "additionalProperties": False, + "type": "object", + "required": ["scope_id"], + }, + "GetTopicMemoryProcessingBlockRequest": { + "properties": { + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, + "block_id": {"type": "string", "maxLength": 36, "minLength": 1}, + }, + "additionalProperties": False, + "type": "object", + "required": ["scope_id", "block_id"], + }, + "RetryTopicMemoryProcessingRequest": { + "properties": { + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, + "block_id": {"type": "string", "maxLength": 36, "minLength": 1}, + }, + "additionalProperties": False, + "type": "object", + "required": ["scope_id", "block_id"], + }, + "TopicMemoryProcessingBlock": { + "properties": { + "block_id": {"type": "string"}, + "attempt_id": {"type": "string"}, + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, + "binding_name": {"type": "string"}, + "source_after": {"type": "integer", "minimum": 0.0}, + "source_through": {"type": "integer", "minimum": 1.0}, + "target_source_through": {"type": "integer", "minimum": 1.0}, + "attempts": {"type": "integer", "minimum": 1.0}, + "requests": {"type": "integer", "minimum": 0.0}, + "tokens": {"type": "integer", "minimum": 0.0}, + "error_code": {"type": "string"}, + "last_error_code": {"type": "string", "nullable": True}, + "blocked_at": {"type": "string", "format": "date-time"}, + "retried_at": {"type": "string", "format": "date-time", "nullable": True}, + "retry_request_generation": {"type": "integer", "minimum": 1.0, "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": [ + "block_id", + "attempt_id", + "scope_id", + "binding_name", + "source_after", + "source_through", + "target_source_through", + "attempts", + "requests", + "tokens", + "error_code", + "last_error_code", + "blocked_at", + "retried_at", + "retry_request_generation", + ], + }, + "TopicMemoryProcessingState": {"type": "string", "enum": ["idle", "pending", "blocked"]}, + "TopicMemoryProcessingRetryStatus": {"type": "string", "enum": ["accepted"]}, + "TopicMemoryProcessingStatus": { + "properties": { + "state": {"$ref": "#/components/schemas/TopicMemoryProcessingState"}, + "source_cursor": {"type": "integer", "minimum": 0.0}, + "source_head": {"type": "integer", "minimum": 0.0}, + "target_source_through": {"type": "integer", "minimum": 0.0, "nullable": True}, + "block": {"allOf": [{"$ref": "#/components/schemas/TopicMemoryProcessingBlock"}], "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["state", "source_cursor", "source_head", "target_source_through", "block"], + }, + "RetryTopicMemoryProcessingResponse": { + "properties": { + "status": {"$ref": "#/components/schemas/TopicMemoryProcessingRetryStatus"}, + "block": {"$ref": "#/components/schemas/TopicMemoryProcessingBlock"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["status", "block"], + }, "FlushTopicMemoryRequest": { "properties": {"scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}}, "additionalProperties": False, diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index 5116b2519d..333806429d 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -131,6 +131,17 @@ TopicMemoryCurrentItem, TopicMemorySearchResult, ) +from powercontext.builtin.artifacts.topic_memory.processing import ( + TopicMemoryBlockConflictError, + TopicMemoryBlockNotFoundError, +) +from powercontext.builtin.artifacts.topic_memory.processing import ( + TopicMemoryProcessingStatus as RuntimeTopicMemoryProcessingStatus, +) +from powercontext.builtin.artifacts.topic_memory.processing import ( + TopicMemoryRetryResult as RuntimeTopicMemoryRetryResult, +) +from powercontext.builtin.artifacts.topic_memory.processing import TopicMemoryWorkBlock as RuntimeTopicMemoryWorkBlock from powercontext.builtin.code.application import CodeApplication from powercontext.builtin.code.errors import CodeError from powercontext.builtin.code.models import CodeQueryRequest as RuntimeCodeQueryRequest @@ -641,11 +652,17 @@ from powercontext.http._generated.models import ( ArtifactTagPage, ArtifactTagSet, + GetTopicMemoryProcessingBlockRequest, QueryArtifactTagsRequest, ReplaceArtifactTagsRequest, + RetryTopicMemoryProcessingRequest, + RetryTopicMemoryProcessingResponse, TaggableArtifactFamily, TagMatch, TagTargetType, + TopicMemoryProcessingBlock, + TopicMemoryProcessingStatus, + TopicMemoryProcessingStatusRequest, ) from powercontext.http._generated.models import ( ShareUnit as TransportShareUnit, @@ -708,6 +725,8 @@ GET_SOURCE, GET_STATS, GET_TOPIC_MEMORY, + GET_TOPIC_MEMORY_PROCESSING_BLOCK, + GET_TOPIC_MEMORY_PROCESSING_STATUS, HANDOFF_CURRENT_WORK, IMPORT_EXTERNAL_SKILL, LIST_ACCESS_AUDIT, @@ -756,6 +775,7 @@ RESOLVE_SCOPE_BINDING, RESOLVE_SCOPE_SELECTION, RETIRE_MEMORY_ENTRY, + RETRY_TOPIC_MEMORY_PROCESSING, REVISE_ARTIFACT_CANDIDATE, REVISE_MEMORY_ENTRY, REVOKE_ACCESS_BINDING, @@ -1206,6 +1226,12 @@ def for_scope(self, scope_id: str, /) -> _ScopedWorkApplication: ... class _ScopedTopicMemoryApplication(Protocol): + async def processing_status(self) -> RuntimeTopicMemoryProcessingStatus: ... + + async def get_block(self, block_id: str) -> RuntimeTopicMemoryWorkBlock: ... + + async def retry(self, block_id: str) -> RuntimeTopicMemoryRetryResult: ... + async def search( self, request: RuntimeSearchTopicMemoryRequest, @@ -1451,6 +1477,9 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: execution_context=_atomic_execution_context, ) _add_route(app, FLUSH_TOPIC_MEMORY, flush_topic_memory) + _add_route(app, GET_TOPIC_MEMORY_PROCESSING_STATUS, get_topic_memory_processing_status) + _add_route(app, GET_TOPIC_MEMORY_PROCESSING_BLOCK, get_topic_memory_processing_block) + _add_route(app, RETRY_TOPIC_MEMORY_PROCESSING, retry_topic_memory_processing) _add_route(app, SEARCH_TOPIC_MEMORY, search_topic_memory) _add_route(app, SEARCH_ARTIFACTS, search_artifacts) _add_route(app, GET_TOPIC_MEMORY, get_topic_memory) @@ -2515,6 +2544,30 @@ async def list_artifacts( ) +async def get_topic_memory_processing_status( + request: TopicMemoryProcessingStatusRequest, + application: Annotated[ServerApplication, Depends(_require_application)], +) -> TopicMemoryProcessingStatus: + result = await application.topic_memory.for_scope(request.scope_id).processing_status() + return TopicMemoryProcessingStatus.model_validate(result.model_dump()) + + +async def get_topic_memory_processing_block( + request: GetTopicMemoryProcessingBlockRequest, + application: Annotated[ServerApplication, Depends(_require_application)], +) -> TopicMemoryProcessingBlock: + result = await application.topic_memory.for_scope(request.scope_id).get_block(request.block_id) + return TopicMemoryProcessingBlock.model_validate(result.model_dump()) + + +async def retry_topic_memory_processing( + request: RetryTopicMemoryProcessingRequest, + application: Annotated[ServerApplication, Depends(_require_application)], +) -> RetryTopicMemoryProcessingResponse: + result = await application.topic_memory.for_scope(request.scope_id).retry(request.block_id) + return RetryTopicMemoryProcessingResponse.model_validate(result.model_dump()) + + async def flush_topic_memory( request: FlushTopicMemoryRequest, application: Annotated[ServerApplication, Depends(_require_application)], @@ -5757,6 +5810,20 @@ def _map_report_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | def _map_domain_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None]: + if isinstance(error, TopicMemoryBlockNotFoundError): + return ( + status.HTTP_404_NOT_FOUND, + "topic_memory_block_not_found", + "The Topic Memory block was not found in this Scope.", + None, + ) + if isinstance(error, TopicMemoryBlockConflictError): + return ( + status.HTTP_409_CONFLICT, + "topic_memory_block_conflict", + "The Topic Memory block no longer owns this frontier.", + None, + ) if isinstance(error, AtomicMemoryIndexError): return 503, error.code, str(error), None if isinstance(error, AtomicMemoryError): diff --git a/tests/builtin/runtime/test_topic_memory_recovery.py b/tests/builtin/runtime/test_topic_memory_recovery.py new file mode 100644 index 0000000000..58c65909b0 --- /dev/null +++ b/tests/builtin/runtime/test_topic_memory_recovery.py @@ -0,0 +1,339 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +import sqlite3 +from dataclasses import replace + +import pytest +from sqlalchemy import insert, update + +from powercontext.builtin.artifacts.topic_memory.generation import TopicMemoryGenerationError +from powercontext.builtin.artifacts.topic_memory.processing import ( + TopicMemoryBlockConflictError, + TopicMemoryBlockNotFoundError, +) +from powercontext.builtin.persistence.processing import ArtifactProcessingPendingRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import TOPIC_MEMORY_WORK_BUDGETS_TABLE as BUDGETS +from powercontext.builtin.persistence.topic_memory_budget import record_topic_memory_work_failure +from powercontext.builtin.runtime.artifact_processing import ArtifactProcessingBinding, ArtifactProcessingSupervisor +from powercontext.builtin.runtime.topic_memory_recovery import TopicMemoryRecoveryService +from powercontext.builtin.runtime.topic_memory_scope import topic_memory_processing_block +from powercontext.sources import SourceMaterialization +from tests.builtin.persistence.contract import NoteSource +from tests.builtin.runtime.test_processing_scheduler import _Launcher, _wait +from tests.builtin.runtime.test_topic_memory_processing import _repositories +from tests.builtin.runtime.test_topic_memory_scope import BINDING, _assignment, _capture, _Probe, _processor, _state +from tests.builtin.runtime.test_topic_memory_terminal_frontier import _budget, _seed + + +def test_inspection_cannot_block_or_reset_an_active_third_attempt(): + async def scenario(): + manager, profile, sources, topics = await _repositories() + release = asyncio.Event() + entered = asyncio.Event() + task = None + try: + await _capture(profile, sources, 1) + assignment = await _assignment(profile) + await _seed(profile.database, attempts=2) + + async def running(_count): + entered.set() + await release.wait() + + processor = _processor(profile, sources, topics, _Probe(running)) + task = asyncio.create_task(processor.process(assignment)) + async with asyncio.timeout(5): + await entered.wait() + service = TopicMemoryRecoveryService(profile.database) + before = await _budget(profile.database) + assert before["attempts"] == 3 + status = await service.status("scope-a") + assert status.state == "pending" and status.block is None + assert await _budget(profile.database) == before + release.set() + assert (await task).outcome.value == "succeeded" + assert (await service.status("scope-a")).state == "idle" + finally: + release.set() + if task is not None: + await task + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +def test_explicit_retry_retains_diagnostics_and_duplicate_cannot_reset_a_new_block(): + async def scenario(): + manager, profile, sources, topics = await _repositories() + try: + await _capture(profile, sources, 1) + assignment = await _assignment(profile) + + async def unavailable(_count): + raise TopicMemoryGenerationError("probe_unavailable") + + processor = _processor(profile, sources, topics, _Probe(unavailable)) + service = TopicMemoryRecoveryService(profile.database) + for _ in range(3): + with pytest.raises(TopicMemoryGenerationError, match="probe_unavailable"): + await processor.process(assignment) + first = (await service.status("scope-a")).block + assert first is not None + assert first.error_code == "window_attempt_limit" + assert first.last_error_code == "probe_unavailable" and first.attempts == 3 + before = await _state(profile) + accepted, duplicate = await asyncio.gather( + service.retry("scope-a", first.block_id), service.retry("scope-a", first.block_id) + ) + assert accepted == duplicate + after = await _state(profile) + assert after[1] == before[1] and after[3] == before[3] + assert after[2].source_through == before[2].source_through + assert after[2].handled_flush_generation == before[2].handled_flush_generation + assert after[2].flush_generation == before[2].flush_generation + 1 + assert after[0].requested_generation == before[0].requested_generation + 1 + assert accepted.block.retried_at is not None + assert accepted.block.blocked_at == first.blocked_at + assignment = replace(assignment, claimed_request_generation=accepted.block.retry_request_generation) + for _ in range(3): + with pytest.raises(TopicMemoryGenerationError, match="probe_unavailable"): + await processor.process(assignment) + second = (await service.status("scope-a")).block + assert second is not None and second.block_id != first.block_id + budget = await _budget(profile.database) + assert await service.retry("scope-a", first.block_id) == accepted + assert await _budget(profile.database) == budget + assert await service.get_block("scope-a", first.block_id) == accepted.block + assert (await service.status("scope-a")).block == second + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +def test_obsolete_final_attempt_can_be_retried_without_restoring_its_publication_authority(): + async def scenario(): + manager, profile, sources, topics = await _repositories() + release = asyncio.Event() + entered = asyncio.Event() + task = None + try: + await _capture(profile, sources, 1) + assignment = await _assignment(profile) + await _seed(profile.database, attempts=2) + + async def running(_count): + entered.set() + await release.wait() + + task = asyncio.create_task(_processor(profile, sources, topics, _Probe(running)).process(assignment)) + async with asyncio.timeout(5): + await entered.wait() + await _assignment(profile) # Revoke the old term before it can publish. + service = TopicMemoryRecoveryService(profile.database) + block = (await service.status("scope-a")).block + assert block is not None and block.attempts == 3 + accepted = await service.retry("scope-a", block.block_id) + release.set() + assert (await task).outcome.value == "leadership_lost" + assert (await _state(profile))[1] == 0 + assert await service.get_block("scope-a", block.block_id) == accepted.block + assert (await service.status("scope-a")).block is None + finally: + release.set() + if task is not None: + await task + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +def test_unretried_stale_block_cannot_reset_replacement_allowance(): + async def scenario(): + manager, profile, sources, _ = await _repositories() + try: + await _capture(profile, sources, 1) + await _seed(profile.database) + service = TopicMemoryRecoveryService(profile.database) + block = (await service.status("scope-a")).block + assert block is not None + async with profile.database.transaction() as connection: + await connection.execute(update(BUDGETS).values(attempt_id="replacement-attempt", attempts=1)) + before = await _budget(profile.database) + with pytest.raises(TopicMemoryBlockConflictError): + await service.retry("scope-a", block.block_id) + assert await _budget(profile.database) == before + assert (await service.get_block("scope-a", block.block_id)).retried_at is None + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +def test_legacy_attempt_ids_are_scoped_and_blocks_cannot_cross_scopes(): + async def scenario(): + manager, profile, sources, _ = await _repositories() + try: + await _capture(profile, sources, 1) + await _seed(profile.database) + before = await _budget(profile.database) + async with profile.database.transaction() as connection: + source = await sources.add( + connection, + "scope-b", + NoteSource(name="other", body="other scope", materialization=SourceMaterialization.CAPTURED), + ) + await ArtifactProcessingPendingRepository().raise_source( + connection, "scope-b", BINDING, source.journal_position + ) + await connection.execute(insert(BUDGETS).values(before | {"scope_id": "scope-b"})) + service = TopicMemoryRecoveryService(profile.database) + a = (await service.status("scope-a")).block + b = (await service.status("scope-b")).block + assert a is not None and b is not None + assert a.block_id != b.block_id and a.attempt_id == b.attempt_id + with pytest.raises(TopicMemoryBlockNotFoundError): + await service.get_block("scope-b", a.block_id) + with pytest.raises(TopicMemoryBlockNotFoundError): + await service.retry("scope-b", a.block_id) + assert (await service.status("scope-b")).block == b + await service.retry("scope-a", a.block_id) + assert (await service.status("scope-b")).block == b + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("custom_error_code", [None, "private provider payload " * 10]) +def test_generic_failed_workers_retain_last_error_and_stop_after_three_attempts(tmp_path, caplog, custom_error_code): + caplog.set_level(logging.WARNING, logger="powercontext.builtin.runtime.artifact_processing") + + class _ProviderError(RuntimeError): + error_code = custom_error_code + + async def scenario(): + manager, profile, sources, topics = await _repositories( + SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'failure.db'}") + ) + try: + await _capture(profile, sources, 1) + + async def unavailable(_count): + raise _ProviderError("private provider payload") # noqa: TRY003 - verify content-free diagnostics + + launcher = _Launcher( + profile.database, on_work=_processor(profile, sources, topics, _Probe(unavailable)).process + ) + async with ArtifactProcessingSupervisor( + database=profile.database, + bindings=( + ArtifactProcessingBinding( + BINDING, + "topic-memory", + launcher, + work_block=topic_memory_processing_block, + work_failure=record_topic_memory_work_failure, + ), + ), + lease_mode="single-process", + retry_base_seconds=0.01, + retry_cap_seconds=0.01, + retry_jitter=lambda: 1, + blocked_check_seconds=0.03, + ) as supervisor: + await _wait( + lambda: any(getattr(r, "event", None) == "artifact_processing.blocked" for r in caplog.records) + ) + service = TopicMemoryRecoveryService(profile.database) + block = (await service.status("scope-a")).block + assert block is not None and block.last_error_code == "processing_failed" + assert block.attempts == 3 and block.error_code == "window_attempt_limit" + await asyncio.sleep(0.1) + assert supervisor.family_status["topic-memory"]["failed"] == 3 + assert (await service.status("scope-a")).block == block + assert "private provider payload" not in block.model_dump_json() + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +def test_concurrent_retries_from_independent_database_connections_grant_one_allowance(tmp_path): + async def scenario(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'concurrent.db'}") + manager_a, profile_a, sources, _ = await _repositories(config) + manager_b = None + try: + await _capture(profile_a, sources, 1) + await _seed(profile_a.database) + manager_b, profile_b, _, _ = await _repositories(config) + service_a = TopicMemoryRecoveryService(profile_a.database) + service_b = TopicMemoryRecoveryService(profile_b.database) + first, second = await asyncio.gather(service_a.status("scope-a"), service_b.status("scope-a")) + block = first.block + assert block is not None and second.block == block + before = await _state(profile_a) + accepted, duplicate = await asyncio.gather( + service_a.retry("scope-a", block.block_id), service_b.retry("scope-a", block.block_id) + ) + assert accepted == duplicate + assert accepted.block.retried_at is not None + after = await _state(profile_a) + assert after[0].requested_generation == before[0].requested_generation + 1 + assert after[2].flush_generation == before[2].flush_generation + 1 + assert after[1] == before[1] and after[3] == before[3] + assert (await service_a.status("scope-a")).block is None + assert await service_b.get_block("scope-a", block.block_id) == accepted.block + finally: + if manager_b is not None: + await manager_b.__aexit__(None, None, None) + await manager_a.__aexit__(None, None, None) + + asyncio.run(scenario()) + + +def test_startup_adds_attempt_table_and_preserves_legacy_budget(tmp_path): + database_path = tmp_path / "legacy.db" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}") + + async def seed(): + manager, profile, sources, _ = await _repositories(config) + try: + await _capture(profile, sources, 1) + await _seed(profile.database) + return await _budget(profile.database) + finally: + await manager.__aexit__(None, None, None) + + before = asyncio.run(seed()) + with sqlite3.connect(database_path) as connection: + connection.execute("DROP TABLE pc_topic_memory_work_attempts") + + async def inspect(): + manager, profile, _, _ = await _repositories(config) + try: + service = TopicMemoryRecoveryService(profile.database) + block = (await service.status("scope-a")).block + assert block is not None and block.last_error_code is None + assert await _budget(profile.database) == before + assert (await service.status("scope-a")).block == block + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(inspect()) diff --git a/tests/e2e/test_topic_memory_recovery_access.py b/tests/e2e/test_topic_memory_recovery_access.py new file mode 100644 index 0000000000..bff822ead9 --- /dev/null +++ b/tests/e2e/test_topic_memory_recovery_access.py @@ -0,0 +1,137 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import threading +from contextlib import asynccontextmanager + +import httpx +import pytest +from pydantic import AnyHttpUrl + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime import InferenceConfig, RuntimeConfig +from powercontext.server.authz.composition import open_builtin_access_control +from powercontext.server.factory import create_server_app +from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings +from tests.builtin.runtime.test_topic_memory_terminal_frontier import _budget, _seed +from tests.e2e.test_access_control_regressions import ADMIN, _Authentication, _grant, _scope, _server +from tests.e2e.test_topic_memory_terminal_frontier import _Handler, _Provider + + +@asynccontextmanager +async def _recovery_server(tmp_path, backend, inference): + if backend == "casbin": + # Casbin's injected provider supports these reads but not spawned processing. + async with _server(tmp_path, backend) as (app, client, _): + yield app, client + return + database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'recovery.db'}") + async with open_builtin_access_control(database, bootstrap_administrators=(ADMIN,), deployment_id="recovery"): + pass + app = create_server_app( + settings=ServerSettings( + database=database, + inference=inference, + runtime=RuntimeConfig(artifact_processing_families=("topic-memory",)), + access=AccessControlConfig(mode="enforced", deployment_id="recovery", background_principal_id=ADMIN.id), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ), + authentication_provider=_Authentication(), + scheduler_path=tmp_path / "scheduler.db", + ) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client, + ): + yield app, client + + +@pytest.mark.parametrize("backend", ["builtin", "casbin"]) +def test_processing_status_is_scoped_and_only_admin_can_grant_retry(tmp_path, monkeypatch, backend): + monkeypatch.setenv("OPENAI_API_KEY", "local-test-provider") + provider = _Provider(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=provider.serve_forever, daemon=True) + thread.start() + + async def scenario(): + async with _recovery_server( + tmp_path, + backend, + inference=InferenceConfig( + generation_model="openai-chat:local-test", + generation_base_url=AnyHttpUrl(f"http://127.0.0.1:{provider.server_port}/v1"), + ), + ) as (app, client): + scope = await _scope(client) + for role in ("scope.admin", "scope.contributor"): + grant = await client.post( + "/v1/access/bindings/create", + json={ + "subject": {"type": ADMIN.type, "id": ADMIN.id}, + "resource": {"type": "scope", "scope_id": scope}, + "role": role, + "idempotency_key": f"background-{role}", + }, + ) + assert grant.status_code == 201, grant.text + captured = await client.post( + "/v1/sources/content", + json={"scope_id": scope, "source_id": "blocked", "content": "retained evidence"}, + ) + assert captured.status_code == 202, captured.text + database = app.state.application._provider.database + await _seed(database, scope_id=scope) + before = await _budget(database) + status_path = "/v1/topic-memory/processing/status" + block_path = "/v1/topic-memory/processing/blocks/get" + retry_path = "/v1/topic-memory/processing/retry" + observed = await client.post(status_path, json={"scope_id": scope}) + assert observed.status_code == 200, observed.text + block = observed.json()["block"] + payload = {"scope_id": scope, "block_id": block["block_id"]} + await _grant(client, scope, "reader", "scope.viewer") + await _grant(client, scope, "writer", "scope.contributor") + await _grant(client, scope, "manager", "scope.admin") + for principal in ("reader", "writer"): + headers = {"Authorization": f"Bearer {principal}"} + assert (await client.post(status_path, headers=headers, json={"scope_id": scope})).status_code == 200 + assert (await client.post(block_path, headers=headers, json=payload)).status_code == 200 + assert (await client.post(retry_path, headers=headers, json=payload)).status_code == 403 + stranger = {"Authorization": "Bearer stranger"} + assert (await client.post(status_path, headers=stranger, json={"scope_id": scope})).status_code == 403 + assert (await client.post(block_path, headers=stranger, json=payload)).status_code == 403 + assert await _budget(database) == before + manager = {"Authorization": "Bearer manager"} + accepted = await client.post(retry_path, headers=manager, json=payload) + if backend == "casbin": + assert accepted.status_code == 503, accepted.text + assert await _budget(database) == before + return + assert accepted.status_code == 200, accepted.text + assert accepted.json()["block"]["blocked_at"] == block["blocked_at"] + assert accepted.json()["block"]["retried_at"] is not None + duplicate = await client.post(retry_path, headers=manager, json=payload) + assert duplicate.status_code == 200 and duplicate.json() == accepted.json() + assert (await client.post(status_path, json={"scope_id": scope})).json()["block"] is None + missing = await client.post(retry_path, headers=manager, json=payload | {"block_id": "missing-block"}) + assert missing.status_code == 404 and missing.json()["error"]["code"] == "topic_memory_block_not_found" + + try: + asyncio.run(scenario()) + finally: + provider.shutdown() + provider.server_close() + thread.join(timeout=5) diff --git a/tests/e2e/test_topic_memory_terminal_frontier.py b/tests/e2e/test_topic_memory_terminal_frontier.py index 0eda4b88e1..e9ebd094e6 100644 --- a/tests/e2e/test_topic_memory_terminal_frontier.py +++ b/tests/e2e/test_topic_memory_terminal_frontier.py @@ -25,18 +25,27 @@ import httpx from pydantic import AnyHttpUrl -from sqlalchemy import insert, select, update +from sqlalchemy import insert, select from powercontext.builtin.artifacts.topic_memory import TOPIC_MEMORY_SOURCE_WINDOW_BINDING as BINDING from powercontext.builtin.persistence.cursors import SourceCursorRepository from powercontext.builtin.persistence.processing import ArtifactProcessingPendingRepository from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository -from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.processing_migration import apply_processing_migration +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES from powercontext.builtin.persistence.tables import TOPIC_MEMORY_WORK_BUDGETS_TABLE as BUDGETS from powercontext.builtin.runtime.artifact_processing import SpawnArtifactProcessingWorkerLauncher -from powercontext.builtin.runtime.config import InferenceConfig, RuntimeConfig +from powercontext.builtin.runtime.config import BuiltinConfig, InferenceConfig, RuntimeConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest from powercontext.client import PowerContextClient -from powercontext.http import CaptureContentSourceRequest, FlushTopicMemoryRequest +from powercontext.http import ( + CaptureContentSourceRequest, + FlushTopicMemoryRequest, + GetTopicMemoryProcessingBlockRequest, + RetryTopicMemoryProcessingRequest, + TopicMemoryProcessingStatusRequest, +) from powercontext.server.factory import create_server_app from powercontext.server.settings import McpConfig, ServerSettings @@ -97,7 +106,22 @@ async def _snapshot(database, scope): return None if budget is None else dict(budget), 0 if cursor is None else cursor.cursor.sequence, pending, intent -def test_http_flush_restart_and_model_change_preserve_terminal_budget_until_operator_repair( +async def _disable_processing(database: SQLiteConfig): + manifest = canonical_processing_manifest( + BuiltinConfig(database=database, runtime=RuntimeConfig(artifact_processing_families=())) + ) + async with SQLiteProfile.open(database, tables=BUILTIN_TABLES) as profile: + for _ in range(10): + async with profile.database.transaction() as connection: + progress = await apply_processing_migration( + connection, config_manifest=manifest, migration_id="disable-processing" + ) + if progress.complete: + break + assert progress.complete + + +def test_http_recovery_preserves_blocks_across_restart_and_processing_mode_changes( # noqa: C901 - one recovery lifecycle tmp_path, monkeypatch, caplog ): caplog.set_level(logging.WARNING, logger="powercontext.builtin.runtime.artifact_processing") @@ -117,17 +141,27 @@ async def observe_start(self, assignment): async def scenario(): scope = None retained = None - for model in ("before-restart", "after-restart"): - app = create_server_app( - settings=ServerSettings( - database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'terminal.db'}"), - inference=InferenceConfig( - generation_model=f"openai-chat:{model}", - generation_base_url=AnyHttpUrl(f"http://127.0.0.1:{provider.server_port}/v1"), - ), - runtime=RuntimeConfig(artifact_processing_families=("topic-memory",)), - mcp=McpConfig(enabled=False), + retained_block = None + accepted = None + retry_request = None + database_config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'terminal.db'}") + for model in ("before-restart", "after-restart", None): + settings = ServerSettings( + database=database_config, + inference=InferenceConfig() + if model is None + else InferenceConfig( + generation_model=f"openai-chat:{model}", + generation_base_url=AnyHttpUrl(f"http://127.0.0.1:{provider.server_port}/v1"), ), + runtime=RuntimeConfig(artifact_processing_families=() if model is None else ("topic-memory",)), + mcp=McpConfig(enabled=False), + ) + if model is None: + # Use the supported offline mode switch before disabling processing. + await _disable_processing(database_config) + app = create_server_app( + settings=settings, scheduler_path=tmp_path / "scheduler.db", ) warnings_before = len([ @@ -140,6 +174,19 @@ async def scenario(): client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) database = app.state.application._provider.database supervisor = app.state.application.artifact_processing_supervisor + if model is None: + assert scope is not None and accepted is not None and retry_request is not None + before = await _snapshot(database, scope) + calls_before = provider.calls + assert await client.retry_topic_memory_processing(retry_request) == accepted + assert ( + await client.get_topic_memory_processing_block( + GetTopicMemoryProcessingBlockRequest(scope_id=scope, block_id=accepted.block.block_id) + ) + ) == accepted.block + assert await _snapshot(database, scope) == before + assert len(launches) == 1 and provider.calls == calls_before + continue if scope is None: scope = (await client.get_default_scope()).scope_id for number in range(2): @@ -186,10 +233,33 @@ async def scenario(): len([r for r in caplog.records if getattr(r, "event", None) == "artifact_processing.blocked"]) == warnings_before + 1 ) + status = await client.get_topic_memory_processing_status( + TopicMemoryProcessingStatusRequest(scope_id=scope) + ) + assert status.state.value == "blocked" + assert (status.source_cursor, status.source_head) == (0, 2) + assert status.block is not None + assert status.block.error_code == "window_attempt_limit" + assert status.block.last_error_code is None + assert (status.block.attempts, status.block.requests, status.block.tokens) == (3, 9, 1000) + if retained_block is None: + retained_block = status.block + assert status.block == retained_block + assert ( + await client.get_topic_memory_processing_block( + GetTopicMemoryProcessingBlockRequest(scope_id=scope, block_id=status.block.block_id) + ) + == retained_block + ) if model == "after-restart": - async with database.transaction() as connection: - await connection.execute(update(BUDGETS).values(attempts=1)) - await client.flush_topic_memory(FlushTopicMemoryRequest(scope_id=scope)) + retry_request = RetryTopicMemoryProcessingRequest(scope_id=scope, block_id=status.block.block_id) + accepted, repeated = await asyncio.gather( + client.retry_topic_memory_processing(retry_request), + client.retry_topic_memory_processing(retry_request), + ) + assert accepted == repeated + assert accepted.block.blocked_at == retained_block.blocked_at + assert accepted.block.retried_at is not None await _wait( lambda supervisor=supervisor: supervisor.family_status["topic-memory"]["completed"] == 1 ) @@ -198,6 +268,18 @@ async def scenario(): assert intent.requested_generation == intent.handled_generation assert len(launches) == 1 and provider.calls == calls_before_flush + 1 assert supervisor.family_status["topic-memory"]["failed"] == 0 + assert await client.retry_topic_memory_processing(retry_request) == accepted + assert ( + await client.get_topic_memory_processing_block( + GetTopicMemoryProcessingBlockRequest(scope_id=scope, block_id=retained_block.block_id) + ) + == accepted.block + ) + assert ( + await client.get_topic_memory_processing_status( + TopicMemoryProcessingStatusRequest(scope_id=scope) + ) + ).state.value == "idle" assert "private source sentinel" not in caplog.text try: diff --git a/tests/test_topic_memory_recovery_cli.py b/tests/test_topic_memory_recovery_cli.py new file mode 100644 index 0000000000..a6f176aa26 --- /dev/null +++ b/tests/test_topic_memory_recovery_cli.py @@ -0,0 +1,95 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from contextlib import asynccontextmanager + +import httpx +import pytest +from typer.testing import CliRunner + +import powercontext.client.cli as client_cli +from powercontext.cli.app import create_cli +from powercontext.client import PowerContextClient + + +@pytest.mark.parametrize("command", ["status", "block", "retry"]) +@pytest.mark.parametrize("json_output", [False, True]) +def test_topic_processing_cli_preserves_exact_block_identity_and_diagnostics(monkeypatch, command, json_output): + block = { + "block_id": "c6561f56-1a60-4888-adf3-14b341ce66d5", + "attempt_id": "59527d38-c24d-4396-a595-32116ddc5c1f", + "scope_id": "project", + "binding_name": "topic-memory-source-window", + "source_after": 2, + "source_through": 3, + "target_source_through": 7, + "attempts": 3, + "requests": 9, + "tokens": 1000, + "error_code": "window_attempt_limit", + "last_error_code": "model_timeout", + "blocked_at": "2026-10-13T01:00:00Z", + "retried_at": None, + "retry_request_generation": None, + } + responses = { + "status": { + "state": "blocked", + "source_cursor": 2, + "source_head": 9, + "target_source_through": 7, + "block": block, + }, + "block": block, + "retry": { + "status": "accepted", + "block": block | {"retried_at": "2026-10-13T01:01:00Z", "retry_request_generation": 8}, + }, + } + paths = { + "status": "/v1/topic-memory/processing/status", + "block": "/v1/topic-memory/processing/blocks/get", + "retry": "/v1/topic-memory/processing/retry", + } + received = [] + + def respond(request): + received.append((request.method, request.url.path, json.loads(request.content))) + return httpx.Response(200, json=responses[command]) + + @asynccontextmanager + async def client_factory(server_url, **kwargs): + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(respond)) as transport, + PowerContextClient(server_url, http_client=transport, **kwargs) as client, + ): + yield client + + monkeypatch.setattr(client_cli, "PowerContextClient", client_factory) + arguments = (["--json"] if json_output else []) + ["topic-memory", command, "--scope-id", "project"] + payload = {"scope_id": "project"} + if command != "status": + arguments += ["--block-id", block["block_id"]] + payload["block_id"] = block["block_id"] + result = CliRunner().invoke(create_cli([]), arguments) + assert result.exit_code == 0, result.output + assert received == [("POST", paths[command], payload)] + assert json.loads(result.output) == responses[command] + + +def test_retry_cli_requires_an_explicit_observed_block_id(): + result = CliRunner().invoke(create_cli([]), ["topic-memory", "retry", "--scope-id", "project"]) + assert result.exit_code == 2 + assert "--block-id" in result.output From 9ed6b80047282d01b389b8c65785b936a9f84f2a Mon Sep 17 00:00:00 2001 From: knqiufan Date: Tue, 13 Oct 2026 11:02:31 +0800 Subject: [PATCH 2/6] fix(ci): sync recovery integrations and acceptance setup --- .github/workflows/e2e-harness.yml | 26 +++++++++++++++ .../plugins/powercontext/hooks/bind_tools.py | 3 ++ .../dsh/plugins/powercontext/lib/index.js | 33 +++++++++++++++++++ tests/test_topic_memory_recovery_cli.py | 3 +- 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 8e36436a0e..9b94190e5b 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -325,6 +325,32 @@ jobs: compose=(docker compose -f e2e/bub/compose.yaml -f e2e/bub/compose.oceanbase.yaml) "${compose[@]}" up --detach --wait --wait-timeout 660 oceanbase container_id=$("${compose[@]}" ps -q oceanbase) + # Full schemas and indexes exceed the slim image's default 1 GB maximum. + docker exec "$container_id" obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + datafile_ready=false + for config_attempt in $(seq 1 30); do + docker exec "$container_id" obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ + > "$RUNNER_TEMP/acceptance-datafile.tsv" + if awk ' + $5 == "datafile_size" && $7 == "4G" { size = 1 } + $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } + $5 == "datafile_next" && $7 == "512M" { increment = 1 } + END { exit !(size && maximum && increment) } + ' "$RUNNER_TEMP/acceptance-datafile.tsv"; then + datafile_ready=true + break + fi + sleep 2 + done + cat "$RUNNER_TEMP/acceptance-datafile.tsv" + if [ "$datafile_ready" != true ]; then + echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 + exit 1 + fi + docker exec "$container_id" obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext -e 'SET GLOBAL recyclebin=OFF;' oceanbase_ip=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container_id") test -n "$oceanbase_ip" export POWERCONTEXT_TEST_OCEANBASE_URL="mysql+aoceanbase://root%40test:powercontext-e2e@${oceanbase_ip}:2881/powercontext?charset=utf8mb4" diff --git a/integrations/codex/plugins/powercontext/hooks/bind_tools.py b/integrations/codex/plugins/powercontext/hooks/bind_tools.py index 91dd7ff22a..8f7761ac09 100644 --- a/integrations/codex/plugins/powercontext/hooks/bind_tools.py +++ b/integrations/codex/plugins/powercontext/hooks/bind_tools.py @@ -65,6 +65,8 @@ "get_skill", "get_skill_package_manifest", "get_topic_memory", + "get_topic_memory_processing_block", + "get_topic_memory_processing_status", "handoff_current_work", "import_external_skill", "list_artifact_candidates", @@ -93,6 +95,7 @@ "rename_remote_skill_target", "resolve_external_skill", "restore_atomic_memory", + "retry_topic_memory_processing", "revise_artifact_candidate", "revoke_remote_skill_target", "scan_external_skills", diff --git a/integrations/dsh/plugins/powercontext/lib/index.js b/integrations/dsh/plugins/powercontext/lib/index.js index d4409ed73b..9fb468a93d 100644 --- a/integrations/dsh/plugins/powercontext/lib/index.js +++ b/integrations/dsh/plugins/powercontext/lib/index.js @@ -685,6 +685,39 @@ const OPERATIONS$1 = { successStatuses: [200], emptyStatuses: [] }, + get_topic_memory_processing_status: { + method: "POST", + path: "/v1/topic-memory/processing/status", + location: "body", + scopeMode: "current", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + get_topic_memory_processing_block: { + method: "POST", + path: "/v1/topic-memory/processing/blocks/get", + location: "body", + scopeMode: "current", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + retry_topic_memory_processing: { + method: "POST", + path: "/v1/topic-memory/processing/retry", + location: "body", + scopeMode: "current", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, search_topic_memory: { method: "POST", path: "/v1/topic-memory/search", diff --git a/tests/test_topic_memory_recovery_cli.py b/tests/test_topic_memory_recovery_cli.py index a6f176aa26..cef192ddb6 100644 --- a/tests/test_topic_memory_recovery_cli.py +++ b/tests/test_topic_memory_recovery_cli.py @@ -17,6 +17,7 @@ import httpx import pytest +from click.utils import strip_ansi from typer.testing import CliRunner import powercontext.client.cli as client_cli @@ -92,4 +93,4 @@ async def client_factory(server_url, **kwargs): def test_retry_cli_requires_an_explicit_observed_block_id(): result = CliRunner().invoke(create_cli([]), ["topic-memory", "retry", "--scope-id", "project"]) assert result.exit_code == 2 - assert "--block-id" in result.output + assert "--block-id" in strip_ansi(result.output) From 8e8b9c6b256306b7585f74f600635da81ed339db Mon Sep 17 00:00:00 2001 From: knqiufan Date: Tue, 13 Oct 2026 15:10:58 +0800 Subject: [PATCH 3/6] fix(topic-memory): preserve worker timeout diagnostics --- .../builtin/runtime/artifact_processing.py | 24 +++---- .../runtime/test_topic_memory_recovery.py | 64 +++++++++++++++++++ 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/src/powercontext/builtin/runtime/artifact_processing.py b/src/powercontext/builtin/runtime/artifact_processing.py index 660cb5d104..b0baa95a11 100644 --- a/src/powercontext/builtin/runtime/artifact_processing.py +++ b/src/powercontext/builtin/runtime/artifact_processing.py @@ -1180,11 +1180,7 @@ async def _reap(self, state: _FamilyState) -> None: # noqa: C901 - distinct Wor self._wait_for_block(state, running.assignment, block) continue if state.binding.work_failure is not None: - code = ( - processing_error_code(error.failure.exception_type, error.failure.error_code) - if isinstance(error, _WorkerExecutionError) - else processing_error_code(type(error), _safe_error_attribute(error, "error_code", "")) - ) + code = _worker_error_code(error) try: async with ( asyncio.timeout(_ADMISSION_PROBE_TIMEOUT_SECONDS), @@ -1306,17 +1302,11 @@ def _next_wake_seconds(self) -> float | None: return None if not deadlines else max(0.001, min(deadlines) - now) def _record_failure(self, state: _FamilyState | None, error: BaseException, stage: str) -> None: - error_code = ( - processing_error_code(error.failure.exception_type, error.failure.error_code) - if isinstance(error, _WorkerExecutionError) - else processing_error_code(type(error)) - ) + error_code = _worker_error_code(error) if stage in {"supervisor", "lease_renewal", "scope_discovery"}: error_code = f"{stage}_failed" elif error_code.startswith("model_") or error_code == "invalid_model_output": stage = "inference" - elif isinstance(error, TimeoutError): - error_code = "worker_timeout" occurred_at = datetime.now(UTC).isoformat() for affected in self._families.values() if state is None else (state,): affected.last_error = error_code @@ -1456,6 +1446,16 @@ def _safe_error_attribute(error: BaseException, name: str, fallback: str) -> str return value if isinstance(value, str) and value else fallback +def _worker_error_code(error: BaseException) -> str: + """Keep live and retained Worker diagnostics in the same controlled category.""" + if isinstance(error, _WorkerExecutionError): + return processing_error_code(error.failure.exception_type, error.failure.error_code) + code = processing_error_code(type(error), _safe_error_attribute(error, "error_code", "")) + if code == "processing_failed" and isinstance(error, TimeoutError): + return "worker_timeout" + return code + + def _worker_failure_category(error: BaseException) -> str: """Bound span attributes without copying exception messages or arbitrary child codes.""" if isinstance(error, asyncio.CancelledError): diff --git a/tests/builtin/runtime/test_topic_memory_recovery.py b/tests/builtin/runtime/test_topic_memory_recovery.py index 58c65909b0..3e17236adb 100644 --- a/tests/builtin/runtime/test_topic_memory_recovery.py +++ b/tests/builtin/runtime/test_topic_memory_recovery.py @@ -25,6 +25,7 @@ TopicMemoryBlockConflictError, TopicMemoryBlockNotFoundError, ) +from powercontext.builtin.inference import InferenceTimeoutError from powercontext.builtin.persistence.processing import ArtifactProcessingPendingRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import TOPIC_MEMORY_WORK_BUDGETS_TABLE as BUDGETS @@ -274,6 +275,69 @@ async def unavailable(_count): asyncio.run(scenario()) +@pytest.mark.parametrize("error_code", ["worker_timeout", "model_timeout"]) +def test_timeout_diagnostics_survive_database_reopen(tmp_path, error_code): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'timeout.db'}") + + async def scenario(): + manager, profile, sources, topics = await _repositories(config) + try: + await _capture(profile, sources, 1) + + operation = "private model operation" + + async def timeout(_count): + if error_code == "model_timeout": + raise InferenceTimeoutError(operation, 0.01) + await asyncio.Event().wait() + + # The launcher is a test double; the Supervisor enforces the real deadline. + launcher = _Launcher( + profile.database, on_work=_processor(profile, sources, topics, _Probe(timeout)).process + ) + async with ArtifactProcessingSupervisor( + database=profile.database, + bindings=( + ArtifactProcessingBinding( + BINDING, + "topic-memory", + launcher, + worker_timeout_seconds=0.5, + work_block=topic_memory_processing_block, + work_failure=record_topic_memory_work_failure, + ), + ), + lease_mode="single-process", + retry_base_seconds=0.01, + retry_cap_seconds=0.01, + retry_jitter=lambda: 1, + blocked_check_seconds=0.03, + ) as supervisor: + await _wait(lambda: supervisor.family_status["topic-memory"]["failed"] == 3, timeout_seconds=10) + service = TopicMemoryRecoveryService(profile.database) + status = await service.status("scope-a") + block = status.block + assert status.state == "blocked" and block is not None + assert (status.source_cursor, status.source_head) == (0, 1) + assert block.attempts == 3 and block.error_code == "window_attempt_limit" + assert supervisor.family_status["topic-memory"]["last_error"] == error_code + assert block.last_error_code == error_code + assert await service.get_block("scope-a", block.block_id) == block + assert operation not in block.model_dump_json() + finally: + await manager.__aexit__(None, None, None) + + manager, profile, _, _ = await _repositories(config) + try: + service = TopicMemoryRecoveryService(profile.database) + assert (await service.status("scope-a")).block == block + assert await service.get_block("scope-a", block.block_id) == block + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + def test_concurrent_retries_from_independent_database_connections_grant_one_allowance(tmp_path): async def scenario(): config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'concurrent.db'}") From 4e7b77f523dd20243c13cab517fc2fd1a0ded414 Mon Sep 17 00:00:00 2001 From: knqiufan Date: Thu, 15 Oct 2026 09:36:01 +0800 Subject: [PATCH 4/6] fix(topic-memory): retain the selected attempt range --- .../persistence/topic_memory_budget.py | 3 +- .../runtime/test_topic_memory_recovery.py | 80 ++++++++++++++++++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/src/powercontext/builtin/persistence/topic_memory_budget.py b/src/powercontext/builtin/persistence/topic_memory_budget.py index 9f203dfdcc..138e609123 100644 --- a/src/powercontext/builtin/persistence/topic_memory_budget.py +++ b/src/powercontext/builtin/persistence/topic_memory_budget.py @@ -269,7 +269,8 @@ async def begin(self) -> None: scope_id=self.scope_id, binding_name=self.binding_name, source_after=self.source_after, - source_through=current["source_through"], + # The frontier range is cumulative; the audit range belongs to this attempt. + source_through=self.source_through, target_source_through=max(self.target_source_through, current["source_through"]), attempts=current["attempts"], requests=current["requests"], diff --git a/tests/builtin/runtime/test_topic_memory_recovery.py b/tests/builtin/runtime/test_topic_memory_recovery.py index 3e17236adb..7050f7c051 100644 --- a/tests/builtin/runtime/test_topic_memory_recovery.py +++ b/tests/builtin/runtime/test_topic_memory_recovery.py @@ -25,14 +25,20 @@ TopicMemoryBlockConflictError, TopicMemoryBlockNotFoundError, ) -from powercontext.builtin.inference import InferenceTimeoutError +from powercontext.builtin.inference import InferenceTimeoutError, character_token_estimator from powercontext.builtin.persistence.processing import ArtifactProcessingPendingRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import TOPIC_MEMORY_WORK_BUDGETS_TABLE as BUDGETS from powercontext.builtin.persistence.topic_memory_budget import record_topic_memory_work_failure from powercontext.builtin.runtime.artifact_processing import ArtifactProcessingBinding, ArtifactProcessingSupervisor +from powercontext.builtin.runtime.topic_memory_processing import ( + TopicMemoryAtomicPublisher, + TopicMemoryProcessor, + TopicMemoryStageSet, + TopicMemoryWindowSelector, +) from powercontext.builtin.runtime.topic_memory_recovery import TopicMemoryRecoveryService -from powercontext.builtin.runtime.topic_memory_scope import topic_memory_processing_block +from powercontext.builtin.runtime.topic_memory_scope import TopicMemoryScopeProcessor, topic_memory_processing_block from powercontext.sources import SourceMaterialization from tests.builtin.persistence.contract import NoteSource from tests.builtin.runtime.test_processing_scheduler import _Launcher, _wait @@ -78,6 +84,76 @@ async def running(_count): asyncio.run(scenario()) +def test_attempt_range_stays_exact_after_context_window_shrinks_and_database_reopens(tmp_path): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'attempt-range.db'}") + + async def scenario(): + manager, profile, sources, topics = await _repositories(config) + try: + for number in (1, 2, 3): + await _capture(profile, sources, number, flush=number == 3) + assignment = await _assignment(profile) + + async def unavailable(_count): + raise TopicMemoryGenerationError("probe_unavailable") + + probe = _Probe(unavailable) + estimator = character_token_estimator() + domain = TopicMemoryProcessor( + database=profile.database, + sources=sources, + topics=topics, + stages=TopicMemoryStageSet( + probe=probe, + global_evolver=probe, + planner=probe, + evolver=probe, + temporary=probe, + reconciler=probe, + estimator=estimator, + input_tokens_limit=100_000, + ), + publisher=TopicMemoryAtomicPublisher(profile.database, sources, topics), + ) + for context_window in (100_000, 1, 1): + processor = TopicMemoryScopeProcessor( + profile.database, + domain, + TopicMemoryWindowSelector( + profile.database, sources, estimator, context_window_tokens=context_window + ), + source_window_limit=2, + ) + with pytest.raises(TopicMemoryGenerationError, match="probe_unavailable"): + await processor.process(assignment) + assert [len(value.evidence) for value in probe.inputs] == [2, 1, 1] + service = TopicMemoryRecoveryService(profile.database) + status = await service.status("scope-a") + block = status.block + assert status.state == "blocked" and block is not None + assert (status.source_cursor, status.source_head, status.target_source_through) == (0, 3, 3) + budget = await _budget(profile.database) + assert budget["source_through"] == 2 and budget["attempts"] == 3 + assert block.attempt_id == budget["attempt_id"] + assert (block.source_after, block.source_through, block.target_source_through) == (0, 1, 3) + assert block.attempts == 3 and block.error_code == "window_attempt_limit" + assert block.last_error_code == "probe_unavailable" + assert await service.get_block("scope-a", block.block_id) == block + finally: + await manager.__aexit__(None, None, None) + + manager, profile, _, _ = await _repositories(config) + try: + service = TopicMemoryRecoveryService(profile.database) + assert (await service.status("scope-a")).block == block + assert await service.get_block("scope-a", block.block_id) == block + assert await _budget(profile.database) == budget + finally: + await manager.__aexit__(None, None, None) + + asyncio.run(scenario()) + + def test_explicit_retry_retains_diagnostics_and_duplicate_cannot_reset_a_new_block(): async def scenario(): manager, profile, sources, topics = await _repositories() From 83c2142f4cbd4bed2130b1f18655167b498803b6 Mon Sep 17 00:00:00 2001 From: knqiufan Date: Thu, 15 Oct 2026 09:58:21 +0800 Subject: [PATCH 5/6] docs(topic-memory): clarify legacy range diagnostics --- docs/en/docs/workflows/topic-memory.md | 2 ++ docs/en/rfcs/1417_topic_memory.md | 2 ++ docs/zh/docs/workflows/topic-memory.md | 1 + docs/zh/rfcs/1417_topic_memory.md | 3 ++- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/en/docs/workflows/topic-memory.md b/docs/en/docs/workflows/topic-memory.md index 69365b320a..47ef978298 100644 --- a/docs/en/docs/workflows/topic-memory.md +++ b/docs/en/docs/workflows/topic-memory.md @@ -79,6 +79,8 @@ The state is `idle`, `pending`, or `blocked`. A block contains its exact `block_ attempt/request/token counters, terminal `error_code`, `last_error_code`, and UTC `blocked_at`. An active third attempt remains pending because it can still finish. For legacy budgets, `blocked_at` is the first observation time and an unknown last error is null. +When a legacy budget has no per-attempt audit record, its range remains the stored cumulative frontier coverage; +the final attempt's selected range cannot be reconstructed from that budget. Correct the cause, then copy the returned block ID into `BLOCK_ID`. A Scope administrator can explicitly grant one new bounded allowance and request processing: diff --git a/docs/en/rfcs/1417_topic_memory.md b/docs/en/rfcs/1417_topic_memory.md index 0bbe36e498..6980742c03 100644 --- a/docs/en/rfcs/1417_topic_memory.md +++ b/docs/en/rfcs/1417_topic_memory.md @@ -761,6 +761,8 @@ target, usage snapshot and last error category. A terminal observation adds a st records its timestamp and accepted request generation without replacing the original diagnostics. Records contain no Source/model text. The additive table is created on supported database startup without altering the budget schema. Legacy rows receive an observation time and nullable historical error; no original failure timestamp is inferred. +Without a per-attempt audit record, a legacy range preserves the budget's cumulative coverage rather than inferring +the final attempt's selected end. Lookups use indexed identities and writes touch one frontier. Normal processing adds at most three attempt records per window allowance; explicitly accepted retries add another bounded allowance. Records are retained for audit. diff --git a/docs/zh/docs/workflows/topic-memory.md b/docs/zh/docs/workflows/topic-memory.md index bd276ee360..22b6a7c6a1 100644 --- a/docs/zh/docs/workflows/topic-memory.md +++ b/docs/zh/docs/workflows/topic-memory.md @@ -99,6 +99,7 @@ powercontext --json topic-memory status --scope-id "$SCOPE_ID" 状态为 `idle`、`pending` 或 `blocked`。阻塞记录包含准确的 `block_id`、已尝试范围、保留的 target、尝试/请求/token 计数、终止原因 `error_code`、最后错误类别 `last_error_code` 和 UTC 时间 `blocked_at`。仍在执行的第三次尝试可能 成功，因此显示为 pending。旧预算的 `blocked_at` 表示首次观察到阻塞的时间，无法追溯的最后错误类别为 null。 +旧预算没有逐次审计记录时，范围保留预算中已有的累计覆盖范围，不能从该预算重建最后一次尝试的实际范围。 修复失败原因后，将返回的阻塞 ID 填入 `BLOCK_ID`。Scope 管理员可显式申请一次新的有限额度，并请求后台处理： diff --git a/docs/zh/rfcs/1417_topic_memory.md b/docs/zh/rfcs/1417_topic_memory.md index 8716c3071c..4974dd5b16 100644 --- a/docs/zh/rfcs/1417_topic_memory.md +++ b/docs/zh/rfcs/1417_topic_memory.md @@ -668,7 +668,8 @@ metadata 重新检查出错时保留已知终止状态、等待间隔和日志 `pc_topic_memory_work_attempts` 为每次尝试保留一条 metadata 记录，包含准确的执行 owner、已尝试范围、target、额度 快照及最后错误类别。首次观察到终止时记录稳定 block ID 和 `blocked_at`；retry 保存重试时间及接受的请求 generation， 不覆盖原诊断。记录不含 Source 或模型原文。正常启动会创建新增表，不修改旧预算表。旧记录使用首次观察时间，缺失的 -历史错误类别为 null，不推测原始失败时间。查询使用索引身份，写入仅作用于一个 frontier；正常每个窗口额度最多产生 +历史错误类别为 null，不推测原始失败时间。没有逐次审计记录的旧范围保留预算的累计覆盖范围，不推测最后一次尝试的 +实际终点。查询使用索引身份，写入仅作用于一个 frontier；正常每个窗口额度最多产生 三条尝试记录，显式接受的 retry 可再申请相同的有限额度。记录保留用于审计。 `pc_topic_memory_work_budgets` 保存 frontier、尝试过的最远终点、不透明 attempt ID、尝试次数、预留请求/token 与有界 From 4cdf119bd434321ebf2c18e98834f0190557bbd7 Mon Sep 17 00:00:00 2001 From: knqiufan Date: Thu, 15 Oct 2026 10:25:54 +0800 Subject: [PATCH 6/6] test(topic-memory): wait for exhausted timeout frontier --- .../runtime/test_topic_memory_recovery.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/builtin/runtime/test_topic_memory_recovery.py b/tests/builtin/runtime/test_topic_memory_recovery.py index 7050f7c051..1b430dd8f9 100644 --- a/tests/builtin/runtime/test_topic_memory_recovery.py +++ b/tests/builtin/runtime/test_topic_memory_recovery.py @@ -352,7 +352,8 @@ async def unavailable(_count): @pytest.mark.parametrize("error_code", ["worker_timeout", "model_timeout"]) -def test_timeout_diagnostics_survive_database_reopen(tmp_path, error_code): +def test_timeout_diagnostics_survive_database_reopen(tmp_path, caplog, error_code): + caplog.set_level(logging.WARNING, logger="powercontext.builtin.runtime.artifact_processing") config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'timeout.db'}") async def scenario(): @@ -368,9 +369,18 @@ async def timeout(_count): await asyncio.Event().wait() # The launcher is a test double; the Supervisor enforces the real deadline. - launcher = _Launcher( - profile.database, on_work=_processor(profile, sources, topics, _Probe(timeout)).process - ) + processor = _processor(profile, sources, topics, _Probe(timeout)) + startup_pending = error_code == "worker_timeout" + + async def run_assignment(assignment): + nonlocal startup_pending + if startup_pending: + startup_pending = False + # A startup timeout must not count as a reserved Topic attempt. + await asyncio.Event().wait() + return await processor.process(assignment) + + launcher = _Launcher(profile.database, on_work=run_assignment) async with ArtifactProcessingSupervisor( database=profile.database, bindings=( @@ -378,7 +388,7 @@ async def timeout(_count): BINDING, "topic-memory", launcher, - worker_timeout_seconds=0.5, + worker_timeout_seconds=2.0, work_block=topic_memory_processing_block, work_failure=record_topic_memory_work_failure, ), @@ -389,13 +399,24 @@ async def timeout(_count): retry_jitter=lambda: 1, blocked_check_seconds=0.03, ) as supervisor: - await _wait(lambda: supervisor.family_status["topic-memory"]["failed"] == 3, timeout_seconds=10) + # Worker failures may precede admission; wait for the exhausted frontier. + await _wait( + lambda: any( + getattr(record, "event", None) == "artifact_processing.blocked" + and getattr(record, "scope", None) == "scope-a" + for record in caplog.records + ), + timeout_seconds=30, + ) service = TopicMemoryRecoveryService(profile.database) status = await service.status("scope-a") block = status.block assert status.state == "blocked" and block is not None assert (status.source_cursor, status.source_head) == (0, 1) assert block.attempts == 3 and block.error_code == "window_attempt_limit" + assert int(supervisor.family_status["topic-memory"]["failed"]) >= block.attempts + if error_code == "worker_timeout": + assert int(supervisor.family_status["topic-memory"]["timeouts"]) >= block.attempts assert supervisor.family_status["topic-memory"]["last_error"] == error_code assert block.last_error_code == error_code assert await service.get_block("scope-a", block.block_id) == block