From 5d1fefb6b3b420a8a2f5badd40fdd558a6a8e7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Fri, 23 Oct 2026 16:52:26 +0800 Subject: [PATCH 01/10] feat(artifact): add managed plugin framework and shared retrieval --- desktop/scripts/generate.mjs | 17 +- desktop/src-tauri/src/transport/api.rs | 2 +- .../src-tauri/src/transport/operations.json | 2 +- desktop/src-tauri/src/transport/wire.rs | 101 +- desktop/ui/src/generated/api.d.ts | 420 ++++++- desktop/ui/src/generated/ipc.ts | 19 +- desktop/ui/src/generated/operations.ts | 2 +- docs/en/docs/operate/troubleshoot.md | 12 +- docs/zh/docs/operate/troubleshoot.md | 12 +- examples/artifact_plugins/README.md | 221 ++++ .../incident_report/plugin.py | 88 ++ .../incident_report/plugin.yml | 12 + .../artifact_plugins/ticket_summary/plugin.py | 86 ++ .../ticket_summary/plugin.yml | 12 + .../powercontext/src/operations.generated.ts | 7 + .../powercontext/src/operations.generated.ts | 7 + .../powercontext/src/operations.generated.ts | 7 + openapi/powercontext.yaml | 262 ++++- src/powercontext/__init__.py | 2 + src/powercontext/artifact_plugins.py | 140 +++ src/powercontext/artifacts/indexes.py | 168 +++ src/powercontext/artifacts/search.py | 10 + .../builtin/artifact_plugin_validation.py | 253 ++++ src/powercontext/builtin/artifacts/search.py | 4 + .../persistence/artifact_governance.py | 30 + .../builtin/persistence/artifact_indexes.py | 1009 ++++++++++++++++ .../persistence/artifact_search_schema.py | 239 ++++ .../persistence/artifact_vector_space.py | 155 +++ .../builtin/persistence/custom_artifacts.py | 87 ++ .../builtin/persistence/lexical_projection.py | 4 +- .../builtin/persistence/processing_intents.py | 8 +- .../builtin/persistence/records.py | 50 +- .../builtin/persistence/schema.py | 22 + .../builtin/persistence/sources.py | 26 +- .../builtin/persistence/supervision.py | 12 +- .../builtin/persistence/tables.py | 32 + src/powercontext/builtin/records.py | 4 +- .../builtin/runtime/artifact_plugins.py | 346 ++++++ .../builtin/runtime/artifact_processing.py | 60 +- .../builtin/runtime/artifact_reads.py | 194 ++++ .../builtin/runtime/artifact_search.py | 5 +- .../builtin/runtime/composition.py | 48 +- .../builtin/runtime/custom_artifact_search.py | 445 ++++++++ src/powercontext/builtin/tags.py | 4 +- src/powercontext/http/__init__.py | 14 + src/powercontext/http/_generated/models.py | 177 ++- .../http/_generated/operations.py | 180 +++ src/powercontext/http/_generated/schema.py | 478 +++++++- src/powercontext/server/app.py | 150 ++- src/powercontext/server/artifact_plugins.py | 616 ++++++++++ src/powercontext/server/authz/profiles.py | 17 + src/powercontext/server/factory.py | 33 + src/powercontext/server/settings.py | 1 + .../persistence/test_artifact_indexes.py | 722 ++++++++++++ .../test_custom_artifact_search.py | 112 ++ .../builtin/persistence/test_mysql_schema.py | 4 +- .../runtime/test_artifact_read_context.py | 267 +++++ tests/e2e/test_artifact_index_backends.py | 96 ++ tests/e2e/test_artifact_plugin_search.py | 403 +++++++ tests/e2e/test_artifact_plugins.py | 1014 +++++++++++++++++ tests/e2e/test_artifact_vector_http.py | 128 +++ tests/test_api_contract.py | 6 +- tests/test_artifact_index_maintenance.py | 257 +++++ tests/test_artifact_plugin_validation.py | 81 ++ 64 files changed, 9256 insertions(+), 146 deletions(-) create mode 100644 examples/artifact_plugins/README.md create mode 100644 examples/artifact_plugins/incident_report/plugin.py create mode 100644 examples/artifact_plugins/incident_report/plugin.yml create mode 100644 examples/artifact_plugins/ticket_summary/plugin.py create mode 100644 examples/artifact_plugins/ticket_summary/plugin.yml create mode 100644 src/powercontext/artifact_plugins.py create mode 100644 src/powercontext/artifacts/indexes.py create mode 100644 src/powercontext/builtin/artifact_plugin_validation.py create mode 100644 src/powercontext/builtin/persistence/artifact_indexes.py create mode 100644 src/powercontext/builtin/persistence/artifact_search_schema.py create mode 100644 src/powercontext/builtin/persistence/artifact_vector_space.py create mode 100644 src/powercontext/builtin/persistence/custom_artifacts.py create mode 100644 src/powercontext/builtin/runtime/artifact_plugins.py create mode 100644 src/powercontext/builtin/runtime/artifact_reads.py create mode 100644 src/powercontext/builtin/runtime/custom_artifact_search.py create mode 100644 src/powercontext/server/artifact_plugins.py create mode 100644 tests/builtin/persistence/test_artifact_indexes.py create mode 100644 tests/builtin/persistence/test_custom_artifact_search.py create mode 100644 tests/builtin/runtime/test_artifact_read_context.py create mode 100644 tests/e2e/test_artifact_index_backends.py create mode 100644 tests/e2e/test_artifact_plugin_search.py create mode 100644 tests/e2e/test_artifact_plugins.py create mode 100644 tests/e2e/test_artifact_vector_http.py create mode 100644 tests/test_artifact_index_maintenance.py create mode 100644 tests/test_artifact_plugin_validation.py diff --git a/desktop/scripts/generate.mjs b/desktop/scripts/generate.mjs index 9e055311af..f71aa78156 100644 --- a/desktop/scripts/generate.mjs +++ b/desktop/scripts/generate.mjs @@ -74,6 +74,13 @@ function collectSchemas(value) { for (const item of Object.values(contract.paths)) for (const op of Object.values(item)) if (wanted.includes(op?.operationId)) collectSchemas(op); +// OpenAPI Generator inlines plain string aliases instead of emitting model files. +const stringAliases = new Set( + [...schemaNames].filter((name) => { + const schema = contract.components.schemas[name]; + return schema.type === "string" && !schema.enum; + }), +); function snakeCase(name) { return name .replace(/([a-z0-9])([A-Z])/g, "$1_$2") @@ -114,7 +121,9 @@ async function generateRustModels() { join(tmpdir(), "powercontext-desktop-openapi-"), ); const output = join(temporary, "generated"); - const modelNames = [...schemaNames].sort(); + const modelNames = [...schemaNames] + .filter((name) => !stringAliases.has(name)) + .sort(); const pnpm = "pnpm"; try { await run( @@ -202,7 +211,11 @@ const rustDeclarations = `\n#[rustfmt::skip]\npub fn declarations(config: &ts_rs ...new Set([...schemaNames, ...rustModelNames]), ] .sort() - .map((name) => ` <${name} as ts_rs::TS>::decl(config),`) + .map((name) => + stringAliases.has(name) + ? ` format!("type ${name} = {};", ::name(config)),` + : ` <${name} as ts_rs::TS>::decl(config),`, + ) .join("\n")}\n ]\n}\n`; const outputs = { "src-tauri/src/transport/wire.rs": diff --git a/desktop/src-tauri/src/transport/api.rs b/desktop/src-tauri/src/transport/api.rs index c48e684176..4f11df3470 100644 --- a/desktop/src-tauri/src/transport/api.rs +++ b/desktop/src-tauri/src/transport/api.rs @@ -418,7 +418,7 @@ impl ServerApi { ) .await?; if result.scope_id != scope - || result.family != ArtifactReadFamily::AtomicMemory + || result.family != "atomic-memory" || result.artifact_id != artifact.artifact_id || result.revision != artifact.revision || result.content.get("schema").and_then(|v| v.as_str()) diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 28ea634551..8e606b9ef2 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "a6758fd35296abefb55ab86b152584a369d5f2deedfee7a3f9822e16ad1edc25", + "contractSha256": "707d5f15ac46929b5ff3494f8f84038818f4af75fab69717da3507e7694c7b3c", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/src-tauri/src/transport/wire.rs b/desktop/src-tauri/src/transport/wire.rs index 73828fceaa..399f03373e 100644 --- a/desktop/src-tauri/src/transport/wire.rs +++ b/desktop/src-tauri/src/transport/wire.rs @@ -369,44 +369,6 @@ pub enum ShareUnit { MemoryEntry, } -#[derive( - Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, TS, Default, -)] -pub enum ArtifactReadFamily { - #[serde(rename = "memory")] - #[default] - Memory, - #[serde(rename = "atomic-memory")] - AtomicMemory, - #[serde(rename = "experience")] - Experience, - #[serde(rename = "skill")] - Skill, - #[serde(rename = "handoff")] - Handoff, - #[serde(rename = "profile")] - Profile, - #[serde(rename = "prompt")] - Prompt, - #[serde(rename = "topic-memory")] - TopicMemory, -} - -impl std::fmt::Display for ArtifactReadFamily { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Memory => write!(f, "memory"), - Self::AtomicMemory => write!(f, "atomic-memory"), - Self::Experience => write!(f, "experience"), - Self::Skill => write!(f, "skill"), - Self::Handoff => write!(f, "handoff"), - Self::Profile => write!(f, "profile"), - Self::Prompt => write!(f, "prompt"), - Self::TopicMemory => write!(f, "topic-memory"), - } - } -} - #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, TS)] #[serde(deny_unknown_fields)] pub struct ArtifactReference { @@ -435,9 +397,19 @@ pub struct ArtifactRevision { #[serde(rename = "scope_id")] pub scope_id: String, #[serde(rename = "family")] - pub family: ArtifactReadFamily, + pub family: String, #[serde(rename = "artifact_id")] pub artifact_id: String, + /// Persisted schema version for a custom Artifact family. + #[ts(optional)] + #[ts(type = "string | null")] + #[serde( + rename = "schema_version", + default, + with = "::serde_with::rust::double_option", + skip_serializing_if = "Option::is_none" + )] + pub schema_version: Option>, #[serde(rename = "revision")] pub revision: i64, #[ts(type = "Record")] @@ -495,7 +467,7 @@ impl ArtifactRevision { #[allow(clippy::too_many_arguments)] pub fn new( scope_id: String, - family: ArtifactReadFamily, + family: String, artifact_id: String, revision: i64, content: std::collections::HashMap, @@ -507,6 +479,7 @@ impl ArtifactRevision { scope_id, family, artifact_id, + schema_version: None, revision, content, sources, @@ -648,6 +621,12 @@ impl std::fmt::Display for AtomicMemoryState { #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, TS)] #[serde(deny_unknown_fields)] pub struct Capabilities { + #[ts(optional)] + #[serde( + rename = "custom_artifact_schemas", + skip_serializing_if = "Option::is_none" + )] + pub custom_artifact_schemas: Option>, #[ts(optional)] #[serde(rename = "prompts", skip_serializing_if = "Option::is_none")] pub prompts: Option>, @@ -713,6 +692,7 @@ impl Capabilities { context_versions: Vec, ) -> Capabilities { Capabilities { + custom_artifact_schemas: None, prompts: None, artifact_dreaming: None, artifact_dreaming_operations: None, @@ -730,6 +710,44 @@ impl Capabilities { } } +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, TS)] +#[serde(deny_unknown_fields)] +pub struct CustomArtifactFamilySchema { + #[serde(rename = "family")] + pub family: String, + #[serde(rename = "schema_version")] + pub schema_version: String, + #[ts(type = "Record")] + #[serde(rename = "content_schema")] + pub content_schema: std::collections::HashMap, + /// Legacy persisted text fields; new plugins use search. + #[serde(rename = "search_fields")] + pub search_fields: Vec, + /// Persistent declarative indexes and supported search modes. + #[ts(type = "Record")] + #[ts(optional)] + #[serde(rename = "search", skip_serializing_if = "Option::is_none")] + pub search: Option>, +} + +impl CustomArtifactFamilySchema { + #[allow(clippy::too_many_arguments)] + pub fn new( + family: String, + schema_version: String, + content_schema: std::collections::HashMap, + search_fields: Vec, + ) -> CustomArtifactFamilySchema { + CustomArtifactFamilySchema { + family, + schema_version, + content_schema, + search_fields, + search: None, + } + } +} + #[derive( Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, TS, Default, )] @@ -1452,7 +1470,7 @@ pub fn declarations(config: &ts_rs::Config) -> Vec { ::decl(config), ::decl(config), ::decl(config), - ::decl(config), + format!("type ArtifactReadFamily = {};", ::name(config)), ::decl(config), ::decl(config), ::decl(config), @@ -1462,6 +1480,7 @@ pub fn declarations(config: &ts_rs::Config) -> Vec { ::decl(config), ::decl(config), ::decl(config), + ::decl(config), ::decl(config), ::decl(config), ::decl(config), diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index 63104a9c46..d2c2bb0b2a 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -241,6 +241,131 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/admin/artifact-plugins": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Artifact plugin registrations */ + get: operations["list_artifact_plugins"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get an Artifact plugin registration */ + get: operations["get_artifact_plugin"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/enable-all": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Enable the currently disabled Artifact plugins + * @description Captures registrations and generations at request start, then accepts each eligible enable independently. Enabled, unloaded, failed and transitioning plugins are skipped. Concurrent changes and validation failures are reported per plugin without rolling back successes. Does not discover or reload plugins; poll each status URL for the asynchronous outcome. + */ + post: operations["enable_all_artifact_plugins"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/reload": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Load or update a Worker-loaded Artifact plugin + * @description Reads the binding's direct child entry under POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR and fixes its resolved immutable directory. The manifest binding_name must match the directory entry and request path. Saves validated configuration and declarations without creating a pipeline. Existing enabled or disabled plugins keep their state; initial, unloaded or failed registrations become disabled. This initial path supports the global SQLite Supervisor. Plugin code is loaded only in child processes. + */ + post: operations["reload_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/enable": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Enable an Artifact plugin */ + post: operations["enable_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/disable": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Disable an Artifact plugin */ + post: operations["disable_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/unload": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Unload an Artifact plugin */ + post: operations["unload_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/capabilities": { parameters: { query?: never; @@ -2573,6 +2698,8 @@ export interface components { scope_id: string; family: components["schemas"]["ArtifactReadFamily"]; artifact_id: string; + /** @description Persisted schema version for a custom Artifact family. */ + schema_version?: string | null; revision: number; sources: components["schemas"]["SourceTypeReference"][]; artifacts: components["schemas"]["ArtifactReference"][]; @@ -2614,6 +2741,8 @@ export interface components { scope_id: string; family: components["schemas"]["ArtifactReadFamily"]; artifact_id: string; + /** @description Persisted schema version for a custom Artifact family. */ + schema_version?: string | null; revision: number; content: { [key: string]: unknown; @@ -2947,7 +3076,58 @@ export interface components { /** @enum {string} */ effect: "review_then_publish" | "review_then_publish_configuration" | "review_then_replace_tags"; }; + CustomArtifactFamilySchema: { + family: string; + schema_version: string; + content_schema: { + [key: string]: unknown; + }; + /** @description Legacy persisted text fields; new plugins use search. */ + search_fields: string[]; + /** @description Persistent declarative indexes and supported search modes. */ + search?: { + [key: string]: unknown; + }; + }; + ReloadArtifactPluginRequest: { + expected_generation: number; + }; + ChangeArtifactPluginStateRequest: { + expected_generation: number; + }; + ArtifactPluginState: { + binding_name: string; + plugin_id: string; + family: string; + binding_generation: number; + /** @enum {string} */ + desired_state: "enabled" | "disabled" | "unloaded"; + /** @enum {string} */ + phase: "activating" | "enabled" | "stopping" | "disabled" | "unloading" | "unloaded" | "failed"; + last_error: string | null; + }; + ArtifactPluginPage: { + items: components["schemas"]["ArtifactPluginState"][]; + }; + EnableArtifactPluginResult: { + binding_name: string; + /** @enum {string} */ + outcome: "accepted" | "skipped" | "failed"; + /** @description Registration status URL for the asynchronous result. */ + status_url: string; + state: components["schemas"]["ArtifactPluginState"]; + /** + * @description Safe per-plugin failure code; null for accepted or skipped entries. + * @enum {string|null} + */ + error: "conflict" | "invalid_plugin" | "unavailable" | null; + }; + EnableAllArtifactPluginsResponse: { + items: components["schemas"]["EnableArtifactPluginResult"][]; + }; Capabilities: { + /** @default [] */ + custom_artifact_schemas: components["schemas"]["CustomArtifactFamilySchema"][]; /** @default {} */ prompts: { [key: string]: components["schemas"]["PromptCapability"]; @@ -4463,8 +4643,32 @@ export interface components { [key: string]: unknown; }; }; + /** @description Existing embedding space identity shared with Memory; no additional modality or version fields. */ + EmbeddingProfile: { + profile_id: string; + model: string; + dimension: number; + /** + * @default l2 + * @enum {string} + */ + distance: "l2"; + /** + * @default unit + * @enum {string} + */ + normalization: "none" | "unit"; + }; SearchArtifactsRequest: { - query: string; + query?: string; + query_vector?: { + profile: components["schemas"]["EmbeddingProfile"]; + vector: number[]; + }; + indexes?: { + text?: string[]; + vector?: string[]; + }; /** @default 10 */ limit: number; mode?: string; @@ -4861,8 +5065,7 @@ export interface components { CaptureStatus: "accepted"; /** @enum {string} */ BaseArtifactFamily: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"; - /** @enum {string} */ - ArtifactReadFamily: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"; + ArtifactReadFamily: string; /** @enum {string} */ StatsPeriod: "today" | "7d" | "30d"; /** @enum {string} */ @@ -5783,6 +5986,217 @@ export interface operations { 503: components["responses"]["Unavailable"]; }; }; + list_artifact_plugins: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Durable registration states. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginPage"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 503: components["responses"]["Unavailable"]; + }; + }; + get_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Durable registration state without plugin configuration secrets. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 503: components["responses"]["Unavailable"]; + }; + }; + enable_all_artifact_plugins: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Per-plugin acceptance, skip or failure results. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EnableAllArtifactPluginsResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 503: components["responses"]["Unavailable"]; + }; + }; + reload_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReloadArtifactPluginRequest"]; + }; + }; + responses: { + /** @description Definition saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + enable_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangeArtifactPluginStateRequest"]; + }; + }; + responses: { + /** @description Target saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + disable_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangeArtifactPluginStateRequest"]; + }; + }; + responses: { + /** @description Target saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + unload_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangeArtifactPluginStateRequest"]; + }; + }; + responses: { + /** @description Target saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; get_capabilities: { parameters: { query?: never; diff --git a/desktop/ui/src/generated/ipc.ts b/desktop/ui/src/generated/ipc.ts index 7474c29357..918255c2a3 100644 --- a/desktop/ui/src/generated/ipc.ts +++ b/desktop/ui/src/generated/ipc.ts @@ -55,9 +55,13 @@ export type AccessResourceType = "server" | "scope" | "artifact"; export type AccessRole = "handoff.viewer" | "handoff.receiver" | "artifact.viewer" | "prompt.user" | "artifact.owner" | "scope.viewer" | "scope.contributor" | "scope.reviewer" | "scope.delegator" | "scope.admin" | "server.observer" | "server.admin"; export type ArtifactDreamingOperationCapability = { operation: DreamOperation, output_kind: OutputKind, effect: Effect, }; export type ArtifactFamilyAccessCapability = { family: string, enabled: boolean, share_unit: ShareUnit, actions: Array, grantable_roles: Array, }; -export type ArtifactReadFamily = "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"; +export type ArtifactReadFamily = string; export type ArtifactReference = { family: string, artifact_id: string, revision: number, }; -export type ArtifactRevision = { scope_id: string, family: ArtifactReadFamily, artifact_id: string, revision: number, content: Record, sources: Array, artifacts: Array, content_digest: string, +export type ArtifactRevision = { scope_id: string, family: string, artifact_id: string, +/** + * Persisted schema version for a custom Artifact family. + */ +schema_version?: string | null, revision: number, content: Record, sources: Array, artifacts: Array, content_digest: string, /** * When this revision was committed. Absent for revisions stored before this field existed. */ @@ -83,7 +87,7 @@ score: number, matched_by: Array, }; export type AtomicMemorySearchMode = "text" | "vector" | "hybrid"; export type AtomicMemoryState = "active" | "forgotten" | "merged" | "retired"; export type BuiltinProfile = "coding" | "conversation"; -export type Capabilities = { prompts?: { [key in string]: PromptCapability }, +export type Capabilities = { custom_artifact_schemas?: Array, prompts?: { [key in string]: PromptCapability }, /** * Whether asynchronous Artifact Dream execution is configured. */ @@ -113,6 +117,15 @@ external_skill_registry?: boolean, */ handoff_generation: boolean, search_modes: Array, context_versions: Array, }; export type Configuration = "configured" | "unconfigured" | "unknown"; +export type CustomArtifactFamilySchema = { family: string, schema_version: string, content_schema: Record, +/** + * Legacy persisted text fields; new plugins use search. + */ +search_fields: Array, +/** + * Persistent declarative indexes and supported search modes. + */ +search?: Record, }; export type DreamOperation = "refine_experience" | "derive_skill" | "revise_skill" | "revise_profile" | "revise_memory" | "revise_topic_memory" | "refresh_handoff" | "revise_prompt" | "revise_tags"; export type Effect = "review_then_publish" | "review_then_publish_configuration" | "review_then_replace_tags"; export type ExtractionBackground = { diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index 9799029173..f8b4a6da17 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "a6758fd35296abefb55ab86b152584a369d5f2deedfee7a3f9822e16ad1edc25"; +export const contractSha256 = "707d5f15ac46929b5ff3494f8f84038818f4af75fab69717da3507e7694c7b3c"; export const operations = { "get_liveness": { "method": "GET", diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index bfbba715b0..c5a98b26e8 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -269,7 +269,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ -f ``` @@ -277,7 +277,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions' \ + --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions,pc_artifact_vector_indexes,pc_artifact_index_changes' \ -f ``` @@ -289,17 +289,21 @@ so the previous database remains available for recovery: -f ``` - After Layer 3 completes successfully, import Profile policy in Layer 4: + After Layer 3 completes successfully, import Profile policy and Artifact search projections in Layer 4: ```bash obloader -D --csv \ - --table 'pc_profile_policies' \ + --table 'pc_profile_policies,pc_artifact_index_coverage,pc_artifact_search_text' \ -f ``` Wait for each invocation to complete successfully before starting the next. Treat any OBLoader error, bad record, or conflict record as a failed restore. Order within a layer is irrelevant because no table in a layer references another table in the same layer. + `pc_artifact_vectors` is SQLite-only. For OceanBase/seekDB, restore registered `pc_av_` row data after Layer 4: + use database initialization to create and validate the restored registered spaces first. Preserve the exact + `space_id` and `build_id` identities; never regenerate them from restored content before importing vector rows. + Keep all writers stopped until the fixed schema, native spaces and restored index state are verified. 6. If the installation has additional PowerContext-managed tables not listed above, these tested layers do not classify them. Inspect their foreign-key constraints and place each table after all of its parents; do not add them to an all-table invocation. diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index a8cd9bd9e7..aa49da3b21 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -260,7 +260,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ -f ``` @@ -268,7 +268,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions' \ + --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions,pc_artifact_vector_indexes,pc_artifact_index_changes' \ -f ``` @@ -280,16 +280,20 @@ collation，但不会包含数据库 URL 或凭据。 -f ``` - 第 3 层成功完成后，导入第 4 层中的 Profile Policy： + 第 3 层成功完成后，导入第 4 层中的 Profile Policy 和 Artifact 检索投影： ```bash obloader -D --csv \ - --table 'pc_profile_policies' \ + --table 'pc_profile_policies,pc_artifact_index_coverage,pc_artifact_search_text' \ -f ``` 每个命令成功完成后才能开始下一层。OBLoader 出现任何错误、bad record 或 conflict record 时，都应判定恢复 失败。同一层内的表互不引用，因此层内顺序无关。 + `pc_artifact_vectors` 仅用于 SQLite。OceanBase/seekDB 在第 4 层之后恢复已登记的 `pc_av_` 行数据： + 先通过数据库初始化创建并校验已恢复登记的空间物理表。 + 保留精确的 `space_id` 和 `build_id`；导入向量行之前，不从内容重新分配这些身份。 + 固定 schema、原生空间和已恢复索引状态全部验证完成前，保持所有写入方停止。 6. 如果安装中还存在上面未列出的 PowerContext 管理表，则这些已测试的层并未对它们分类。检查其外键约束，将每张 表放在其所有父表之后；不要把它们加入全表导入命令。 7. 逐表比较源数据库和目标数据库的记录数，检查 identity column collation，并测试仅大小写或重音不同的 diff --git a/examples/artifact_plugins/README.md b/examples/artifact_plugins/README.md new file mode 100644 index 0000000000..70526c943d --- /dev/null +++ b/examples/artifact_plugins/README.md @@ -0,0 +1,221 @@ +# 用户自定义 Artifact 插件示例 + +用户把插件部署到统一发现目录。PowerContext Server 启动时校验新插件并保存定义，首次加载为 `disabled`；管理员启用后，Supervisor 按清单参数派发 Worker。Worker 加载 pipeline，框架保存生成结果，应用通过公开 API 读取。 + +| 模板目录 | family / binding | 生成结果 | 业务配置 | +| --- | --- | --- | --- | +| `incident_report` | `incident-report` / `incident-reports` | 每条 Source 一份工单报告 | `title_prefix` | +| `ticket_summary` | `ticket-summary` / `ticket-summaries` | 每个 Source 窗口一份文本摘要 | `language` | + +本示例使用文件型 SQLite、global Supervisor、Server 的 all 角色、Worker 内加载 Python 和声明式文本检索，不依赖 LLM。内存型 SQLite 可以加载声明，但启用生成会返回 `422`，因为 Worker 子进程无法共享 Server 的内存数据库。 + +## 1. 配置并启动 Server + +在仓库根目录执行： + +```bash +uv sync --extra server --extra cli +export POWERCONTEXT_HOME="$PWD/.powercontext/artifact-plugin-example" +export POWERCONTEXT_SERVER_DATABASE='{"kind":"sqlite","url":"sqlite+aiosqlite:///.powercontext/artifact-plugin-example.db"}' +export POWERCONTEXT_SERVER_RUNTIME='{"artifact_processing_families":[]}' +export POWERCONTEXT_SERVER_AUTH_ENABLED=true +export POWERCONTEXT_SERVER_AUTH_TOKEN=local-example-admin +export POWERCONTEXT_SERVER_ACCESS_MODE=enforced +export POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR="$POWERCONTEXT_HOME/plugins" +mkdir -p "$POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR" +uv run powercontext server run --no-env-file --host 127.0.0.1 --port 8000 +``` + +Server 的 `.env` / envfile 保存数据库、监听、鉴权、共用模型和统一插件目录等全局设置。`POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR` 沿用现有加载规则：环境变量优先于 envfile。各插件的 Scope、批次、调度和业务参数只保存在各自的 `plugin.yml`。 + +以下命令在仓库根目录的另一个终端运行，需要 `curl` 和 `jq`。首次使用先创建 Scope；已有 Scope 的部署可以在启动前准备好插件目录。 + +```bash +export PC_URL=http://127.0.0.1:8000 +export PC_AUTH='Authorization: Bearer local-example-admin' +export PC_PLUGIN_ROOT="$PWD/.powercontext/artifact-plugin-example/plugins" +export PC_VERSION_ROOT="$PWD/.powercontext/artifact-plugin-example/versions" +export PC_REPORT_SCOPE=$(curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/scopes" -d '{"title":"Reports","summary":"Incident reports","idempotency_key":"artifact-reports-example"}' \ + | jq -r .scope_id) +export PC_SUMMARY_SCOPE=$(curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/scopes" -d '{"title":"Summaries","summary":"Ticket summaries","idempotency_key":"artifact-summaries-example"}' \ + | jq -r .scope_id) +``` + +## 2. 部署插件清单和代码 + +每个插件目录有一份完整清单。报告插件示例如下，显式填写的 `scope_ids` 必须对应实际存在的 Scope： + +```yaml +id: example-incident-report +version: "1" +entry: ./plugin.py:create_pipeline +binding_name: incident-reports +scope_ids: [team-a] +max_sources: 20 +processing: + interval_seconds: 1 + max_workers: 1 + timeout_seconds: 60 +config: + title_prefix: 工单快照报告 +``` + +`entry` 相对清单所在目录解析。统一目录只发现直接子目录，目录项名称必须等于 `binding_name`。版本目录放在发现目录之外，通过符号链接提供当前入口；Server 保存解析后的实际目录，切换链接不会改变已有 Worker 使用的代码。 + +Scope、窗口上限、发现间隔、并行 Worker 数和 Worker 超时只作用于对应 binding。数值必须为正数，批次和 Worker 数必须为整数。`config` 按插件的 `Config` 模型校验后交给 pipeline。清单顶层禁止 `enable` / `enabled` 字段，无论值为 true 还是 false；运行状态由管理 API 控制。含这些字段的清单会被拒绝，reload 返回 `422` 并保留已有有效定义和状态。 + +`scope_ids` 可省略：框架动态发现已有输入积压和后续新增的 Scope，并按后台身份校验访问权限。显式填写非空名单时，该 binding 只处理名单内的已有 Scope。处理范围不授予权限。空数组 `scope_ids: []` 会被拒绝。 + +摘要插件使用自己的 Scope、`max_sources=4`、5 秒间隔、2 个 Worker、90 秒超时和 `language` 配置。插件通过安装包中的 `powercontext.artifact_plugins` 声明 schema、输入契约和可选 FTS 能力。 + +```bash +mkdir -p "$PC_PLUGIN_ROOT" "$PC_VERSION_ROOT" +cp -R examples/artifact_plugins/incident_report "$PC_VERSION_ROOT/incident-report-v1" +cp -R examples/artifact_plugins/ticket_summary "$PC_VERSION_ROOT/ticket-summary-v1" +uv run python - <<'PY' +import os +from pathlib import Path +import yaml + +root = Path(os.environ["PC_VERSION_ROOT"]) +for name, scope in ( + ("incident-report-v1", os.environ["PC_REPORT_SCOPE"]), + ("ticket-summary-v1", os.environ["PC_SUMMARY_SCOPE"]), +): + path = root / name / "plugin.yml" + manifest = yaml.safe_load(path.read_text(encoding="utf-8")) + manifest["scope_ids"] = [scope] + path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8") +PY +ln -s "$PC_VERSION_ROOT/incident-report-v1" "$PC_PLUGIN_ROOT/incident-reports" +ln -s "$PC_VERSION_ROOT/ticket-summary-v1" "$PC_PLUGIN_ROOT/ticket-summaries" +``` + +Artifact 插件通过输入映射选择每种 Source 的 Projection。两个示例均消费主线已有的 `content` Source： + +```python +inputs = InputContract(projections={"content": TEXT_EVIDENCE_PROJECTION_KEY}) +``` + +映射必须非空，每种类型选择一个合法的 ProjectionKey。`GenerationSource` 包含 `ref`、`projection` 和完整 JSON `value`；对象、数组、嵌套字段、数字、布尔值和 null 保持原有结构。示例调用 `TextEvidence.model_validate(source.value).content` 读取文本。 + +一个 family 可消费多个 Source 类型，一个类型也可供多个 family 消费。关联由上述输入声明和输出 family 决定，与两端插件 ID 或 binding 名称无关。`max_sources` 限制 journal 窗口条目数；未声明类型被过滤，batch 不保证所有类型齐全，不等待凑齐或自动配对。draft 只列出实际使用的精确 SourceRef。 + +对于外部 Source，Worker 读取已保存的 Projection，不加载 Source 插件代码，不请求外部最新数据。已接受类型缺少指定名称或版本时整个窗口失败，游标不推进，也不使用其他投影代替。输入预算按完整批次 JSON 证据计量，当前上限为 1 MiB；超限不截断或部分提交，可通过新目录和 reload 调整窗口大小。 + +## 3. 加载并启用 + +启动扫描会加载首次发现的插件。Server 已运行时，新增插件需要显式 reload；请求只携带 `expected_generation`，首次为 0： + +```bash +for binding in incident-reports ticket-summaries; do + curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/admin/artifact-plugins/$binding/reload" -d '{"expected_generation":0}' +done +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/admin/artifact-plugins" | jq +curl -fsS -X POST -H "$PC_AUTH" "$PC_URL/v1/admin/artifact-plugins/enable-all" | jq +``` + +reload 返回 `202` 和 `Location`。首次加载只校验入口、Config、输入输出声明、Scope 与身份，保存完整定义和读取能力；不调用生成工厂或生成方法。查询状态到 `disabled` 后即可启用。 + +enable-all 无需请求体，只处理请求开始时已加载且 disabled 的插件，不扫描目录。逐插件返回 `accepted`、`skipped` 或 `failed`、安全错误码和 `status_url`。一个失败不回滚其他成功项；已启用、卸载、failed 或转换中的插件跳过。通过状态 URL 查询到 `phase=enabled` 才表示异步装配完成。首次加载失败不产生注册记录，启动日志报告该目录的失败，其他合法插件继续加载。 + +## 4. 写入 Source 并读取结果 + +```bash +for scope in "$PC_REPORT_SCOPE" "$PC_SUMMARY_SCOPE"; do + jq -n --arg scope "$scope" \ + '{scope_id:$scope,source_id:"ticket-1",content:"Outage: database recovered after retry."}' \ + | curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/sources/content" --data-binary @- +done +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report" | jq +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/scopes/$PC_SUMMARY_SCOPE/artifacts/ticket-summary" | jq +curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report/search" -d '{"query":"Outage"}' | jq +export PC_ARTIFACT=$(curl -fsS -H "$PC_AUTH" \ + "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report" | jq -r '.items[0].artifact_id') +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report/$PC_ARTIFACT" | jq +``` + +生成异步执行，尚无产物时 list / search 可返回空结果；已有产物但首次索引尚未就绪时，search 返回 `503`，稍后重试。list 返回 Artifact 身份，get 返回内容、schema 版本和精确 lineage。启用时从持久游标消费历史与停用期间积压；重复写入相同 Source 不产生重复修订，生成失败不推进游标或提交部分结果。 + +两个示例通过 `ArtifactFamily.search` 和 `TextIndex` 声明检索字段。文本检索支持中文、英文和混合文本，只匹配当前 revision 对应 schema 声明的字段。统一 `/search` 返回 `results`，包含完整 `content`、精确 revision 和 lineage；`include_scores` 可附带评分。 + +Server 根据持久声明恢复索引，不要求生成插件启用或代码目录存在。首次构建尚未就绪时返回 `503`；已发布索引随写入同步维护。检索表由统一数据库初始化创建，无需单独安装。 + +向量、多片段及 hybrid 通过 `ArtifactSearchDeclaration`、`VectorIndex`（`source="text"` 或 `source="content"`） 声明。OceanBase/seekDB 热加载新向量 profile 后，需要重新初始化数据库（例如重启 Server）以创建对应空间。 + +## 5. 启停、更新、卸载与恢复 + +单插件管理使用最新 GET 返回的 `binding_generation`，过期代次返回 `409`： + +```bash +export PC_GENERATION=$(curl -fsS -H "$PC_AUTH" \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports" | jq -r .binding_generation) +curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports/disable" \ + -d "{\"expected_generation\":$PC_GENERATION}" +``` + +查询到 `disabled` 后再读取新代次，可调用 `enable` 或 `unload`。启停不修改清单。卸载回收 Worker 并移除执行定义，保留身份、代次、schema、读取能力、消费进度和已有结果；确认 `unloaded` 后可删除专属版本目录，再次加载需 reload。 + +更新代码或配置时准备完整的新版本目录，再切换发现入口并调用 reload： + +```bash +cp -R "$PC_VERSION_ROOT/incident-report-v1" "$PC_VERSION_ROOT/incident-report-v2" +uv run python - <<'PY' +import os +from pathlib import Path +import yaml + +path = Path(os.environ["PC_VERSION_ROOT"]) / "incident-report-v2/plugin.yml" +manifest = yaml.safe_load(path.read_text(encoding="utf-8")) +manifest["version"] = "2" +manifest["config"]["title_prefix"] = "新版工单报告" +path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8") +PY +ln -s "$PC_VERSION_ROOT/incident-report-v2" "$PC_VERSION_ROOT/incident-reports-next" +mv -Tf "$PC_VERSION_ROOT/incident-reports-next" "$PC_PLUGIN_ROOT/incident-reports" +export PC_GENERATION=$(curl -fsS -H "$PC_AUTH" \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports" | jq -r .binding_generation) +curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports/reload" \ + -d "{\"expected_generation\":$PC_GENERATION}" +``` + +候选校验失败保留原定义和状态。接受更新后撤销旧代次提交资格，回收旧 Worker，再装配新定义；确认完成前保留旧目录。reload 更新正常插件保留原启停状态；卸载或 failed 后加载为 disabled，需要再次启用。同一 `(family, schema_version)` 的内容 schema 不可变。已有检索 family 的 schema 身份、索引定义和版本冻结；无检索 family 可以演进内容 schema，但不能补加检索。允许调整 `modes`、`default_mode`、`defaults`、`candidate_limit`、`rrf_k`、`text_weight`、`vector_weight` 查询策略，更新不会触发索引重建、停搜或重新编码。 + +Server 重启恢复持久快照与已保存的启停、卸载状态，并继续未完成转换。扫描只发现新插件，不覆盖已有配置或重新加载卸载记录；failed 等待管理员 reload 修复。恢复 enabled 会更新代次，管理操作始终先 GET。无需每次重启调用 enable-all，结果读取不依赖插件进程。 + +## 6. 通用检索与精确证据读取 + +上层应用 / Agent 通过公开的 Artifact get / list / 精确 revision 和 FTS 接口获取数据，自行理解问题、组合证据并计算距离、次数或排名。PowerContext 提供存储、生成、版本、鉴权、检索和精确证据读取，不执行插件业务查询回调。 + +生成插件通过 `GenerationContext` 使用同一套读取与检索语义： + +| 接口 | 语义 | +| --- | --- | +| `read_latest(family, artifact_id)` | 读取当前 revision，缺失时返回 `None`；独立调用不形成跨 family 快照 | +| `read_revision(ref)` | 读取精确 revision，支持已被新 head 替代的历史内容；缺失时报错 | +| `list_artifacts(family, limit=50, cursor=None)` | 分页返回可见 active head 的引用；每页上限 100，游标只在本次生成上下文内使用 | +| `search_artifacts(family, query=None, **parameters)` | 经共享 ArtifactSearch 返回完整 Artifact，支持已声明模式和索引；limit 默认 10，上限 200 | + +生成默认可读自己的输出 family；需要跨 family 时，使用 `InputContract(..., artifact_families=(...))` 声明额外范围。所有读取固定在当前 Scope，并受 Worker 身份权限约束。get / list 支持已注册 Artifact family；内置 family 的快照没有自定义 `schema_version` 时返回 `None`。生成 Worker 与 HTTP / SDK 遵守相同的检索语义；无检索能力的已知 family 明确返回不支持。 + +list / search 不自动登记为生成证据。search 返回完整正文；需要用于 draft 的 lineage 或更新基准时，插件仍须显式调用 read_revision 读取所选精确版本，登记实际使用的证据。完整枚举应遍历分页 list 或持久清单，不能用 top-k 表示完整集合。历史读取先固定发布根，再沿清单读取精确引用；当前 FTS 不包含旧 revision，不能据此排除历史对象。 + +统一 HTTP search 复用共享检索服务。插件停用或卸载后，已保存内容的 get / list / 声明式检索继续可用。普通读取与检索不推进 Source 消费进度。 + +## 端到端验收 + +```bash +uv run pytest tests/e2e/test_artifact_plugins.py \ + tests/e2e/test_artifact_plugin_search.py -q +``` + +测试启动安装包提供的 `powercontext server run`，通过真实 envfile / 环境变量、部署目录和 HTTP 完成加载、启用、消费与查询。覆盖首次加载不生成、多插件独立配置、批量部分失败和并发冲突、reload、单插件启停 / 卸载、重启与失败恢复。 + +测试只使用产品的公开路径。 diff --git a/examples/artifact_plugins/incident_report/plugin.py b/examples/artifact_plugins/incident_report/plugin.py new file mode 100644 index 0000000000..c59aa6b608 --- /dev/null +++ b/examples/artifact_plugins/incident_report/plugin.py @@ -0,0 +1,88 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Generate one deterministic report for each exact ticket snapshot.""" + +import hashlib +import json + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.artifact_plugins import ( + ArtifactDraft, + ArtifactFamily, + ArtifactSearchDeclaration, + GenerationBatch, + GenerationContext, + InputContract, + TextIndex, +) +from powercontext.sources import TEXT_EVIDENCE_PROJECTION_KEY, TextEvidence + +plugin_api_version = "1" + + +class Config(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + title_prefix: str = "Incident report" + fail_on_text: str | None = None + + +class IncidentReport(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + title: str = Field(min_length=1) + evidence_text: str = Field(min_length=1) + + +families = ( + ArtifactFamily( + family="incident-report", + schema_version="1", + content_model=IncidentReport, + search=ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="lexical", projection_version="1", fields=("title", "evidence_text")),), + modes=("text",), + default_mode="text", + ), + ), +) +inputs = InputContract( + projections={ + "content": TEXT_EVIDENCE_PROJECTION_KEY, + } +) + + +class IncidentReportPipeline: + def __init__(self, config: Config) -> None: + self.config = config + + async def generate(self, batch: GenerationBatch) -> list[ArtifactDraft]: + drafts = [] + for source in batch.sources: + evidence = TextEvidence.model_validate(source.value) + if self.config.fail_on_text and self.config.fail_on_text in evidence.content: + raise ValueError("Demo generation failure: no drafts from this window will be committed") # noqa: TRY003 + identity = json.dumps(source.ref.model_dump(mode="json"), sort_keys=True, separators=(",", ":")) + drafts.append( + ArtifactDraft( + family="incident-report", + schema_version="1", + artifact_id=hashlib.sha256(identity.encode()).hexdigest(), + content=IncidentReport( + title=f"{self.config.title_prefix}: {source.ref.source_id}", + evidence_text=evidence.content, + ).model_dump(mode="json"), + source_refs=(source.ref,), + ) + ) + return drafts + + +async def create_pipeline(ctx: GenerationContext, config: Config) -> IncidentReportPipeline: + return IncidentReportPipeline(config) diff --git a/examples/artifact_plugins/incident_report/plugin.yml b/examples/artifact_plugins/incident_report/plugin.yml new file mode 100644 index 0000000000..81ebf7cac3 --- /dev/null +++ b/examples/artifact_plugins/incident_report/plugin.yml @@ -0,0 +1,12 @@ +id: example-incident-report +version: "1" +entry: ./plugin.py:create_pipeline +binding_name: incident-reports +scope_ids: [team-a] +max_sources: 20 +processing: + interval_seconds: 1 + max_workers: 1 + timeout_seconds: 60 +config: + title_prefix: 工单快照报告 diff --git a/examples/artifact_plugins/ticket_summary/plugin.py b/examples/artifact_plugins/ticket_summary/plugin.py new file mode 100644 index 0000000000..bec79ba338 --- /dev/null +++ b/examples/artifact_plugins/ticket_summary/plugin.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Summarize each Source window without an external model dependency.""" + +import hashlib +import json + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.artifact_plugins import ( + ArtifactDraft, + ArtifactFamily, + ArtifactSearchDeclaration, + GenerationBatch, + GenerationContext, + InputContract, + TextIndex, +) +from powercontext.sources import TEXT_EVIDENCE_PROJECTION_KEY, TextEvidence + +plugin_api_version = "1" + + +class Config(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + language: str = Field(default="简体中文", min_length=1) + + +class TicketSummary(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + language: str + source_ids: list[str] = Field(min_length=1) + summary_text: str = Field(min_length=1) + + +families = ( + ArtifactFamily( + family="ticket-summary", + schema_version="1", + content_model=TicketSummary, + search=ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="lexical", projection_version="1", fields=("summary_text",)),), + modes=("text",), + default_mode="text", + ), + ), +) +inputs = InputContract( + projections={ + "content": TEXT_EVIDENCE_PROJECTION_KEY, + } +) + + +class TicketSummaryPipeline: + def __init__(self, config: Config) -> None: + self.config = config + + async def generate(self, batch: GenerationBatch) -> list[ArtifactDraft]: + refs = tuple(source.ref for source in batch.sources) + identity = json.dumps([ref.model_dump(mode="json") for ref in refs], sort_keys=True, separators=(",", ":")) + return [ + ArtifactDraft( + family="ticket-summary", + schema_version="1", + artifact_id=hashlib.sha256(identity.encode()).hexdigest(), + content=TicketSummary( + language=self.config.language, + source_ids=[ref.source_id for ref in refs], + summary_text="\n".join( + TextEvidence.model_validate(source.value).content for source in batch.sources + ), + ).model_dump(mode="json"), + source_refs=refs, + ) + ] + + +async def create_pipeline(ctx: GenerationContext, config: Config) -> TicketSummaryPipeline: + return TicketSummaryPipeline(config) diff --git a/examples/artifact_plugins/ticket_summary/plugin.yml b/examples/artifact_plugins/ticket_summary/plugin.yml new file mode 100644 index 0000000000..dbdecc1ecc --- /dev/null +++ b/examples/artifact_plugins/ticket_summary/plugin.yml @@ -0,0 +1,12 @@ +id: example-ticket-summary +version: "1" +entry: ./plugin.py:create_pipeline +binding_name: ticket-summaries +scope_ids: [team-b] +max_sources: 4 +processing: + interval_seconds: 5 + max_workers: 2 + timeout_seconds: 90 +config: + language: 简体中文 diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index 4d3805f2e8..20f1e9a616 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -31,6 +31,13 @@ export const OPERATIONS = { get_liveness: { method: 'GET', path: '/health/live', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_readiness: { method: 'GET', path: '/health/ready', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_server_info: { method: 'GET', path: '/v1/server-info', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { method: 'GET', path: '/v1/admin/artifact-plugins', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_artifact_plugin: { method: 'GET', path: '/v1/admin/artifact-plugins/{binding_name}', location: null, scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + enable_all_artifact_plugins: { method: 'POST', path: '/v1/admin/artifact-plugins/enable-all', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + reload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/reload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + enable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/enable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + disable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/disable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + unload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/unload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, get_capabilities: { method: 'GET', path: '/v1/capabilities', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_scopes: { method: 'GET', path: '/v1/scopes', location: "query", scopeMode: 'none', pathParameters: [], queryParams: ['query','query_field','parent_scope_id','external_reference_kind','binding_integration','binding_kind','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_scope: { method: 'POST', path: '/v1/scopes', location: "body", scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index 4d3805f2e8..20f1e9a616 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -31,6 +31,13 @@ export const OPERATIONS = { get_liveness: { method: 'GET', path: '/health/live', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_readiness: { method: 'GET', path: '/health/ready', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_server_info: { method: 'GET', path: '/v1/server-info', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { method: 'GET', path: '/v1/admin/artifact-plugins', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_artifact_plugin: { method: 'GET', path: '/v1/admin/artifact-plugins/{binding_name}', location: null, scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + enable_all_artifact_plugins: { method: 'POST', path: '/v1/admin/artifact-plugins/enable-all', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + reload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/reload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + enable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/enable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + disable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/disable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + unload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/unload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, get_capabilities: { method: 'GET', path: '/v1/capabilities', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_scopes: { method: 'GET', path: '/v1/scopes', location: "query", scopeMode: 'none', pathParameters: [], queryParams: ['query','query_field','parent_scope_id','external_reference_kind','binding_integration','binding_kind','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_scope: { method: 'POST', path: '/v1/scopes', location: "body", scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index 4d3805f2e8..20f1e9a616 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -31,6 +31,13 @@ export const OPERATIONS = { get_liveness: { method: 'GET', path: '/health/live', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_readiness: { method: 'GET', path: '/health/ready', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_server_info: { method: 'GET', path: '/v1/server-info', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { method: 'GET', path: '/v1/admin/artifact-plugins', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_artifact_plugin: { method: 'GET', path: '/v1/admin/artifact-plugins/{binding_name}', location: null, scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + enable_all_artifact_plugins: { method: 'POST', path: '/v1/admin/artifact-plugins/enable-all', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + reload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/reload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + enable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/enable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + disable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/disable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + unload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/unload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, get_capabilities: { method: 'GET', path: '/v1/capabilities', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_scopes: { method: 'GET', path: '/v1/scopes', location: "query", scopeMode: 'none', pathParameters: [], queryParams: ['query','query_field','parent_scope_id','external_reference_kind','binding_integration','binding_kind','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_scope: { method: 'POST', path: '/v1/scopes', location: "body", scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index c15468cf31..925e56d002 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -527,6 +527,167 @@ paths: $ref: "#/components/responses/Forbidden" "503": $ref: "#/components/responses/Unavailable" + /v1/admin/artifact-plugins: + get: + tags: [artifact-plugins] + summary: List Artifact plugin registrations + operationId: list_artifact_plugins + x-powercontext-access: {action: server.admin, resource: {type: server}} + responses: + "200": + description: Durable registration states. + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginPage'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}: + get: + tags: [artifact-plugins] + summary: Get an Artifact plugin registration + operationId: get_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + responses: + "200": + description: Durable registration state without plugin configuration secrets. + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/enable-all: + post: + tags: [artifact-plugins] + summary: Enable the currently disabled Artifact plugins + description: Captures registrations and generations at request start, then accepts each eligible enable independently. Enabled, unloaded, failed and transitioning plugins are skipped. Concurrent changes and validation failures are reported per plugin without rolling back successes. Does not discover or reload plugins; poll each status URL for the asynchronous outcome. + operationId: enable_all_artifact_plugins + x-powercontext-access: {action: server.admin, resource: {type: server}} + responses: + "200": + description: Per-plugin acceptance, skip or failure results. + content: + application/json: + schema: {$ref: '#/components/schemas/EnableAllArtifactPluginsResponse'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/reload: + post: + tags: [artifact-plugins] + summary: Load or update a Worker-loaded Artifact plugin + description: Reads the binding's direct child entry under POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR and fixes its resolved immutable directory. The manifest binding_name must match the directory entry and request path. Saves validated configuration and declarations without creating a pipeline. Existing enabled or disabled plugins keep their state; initial, unloaded or failed registrations become disabled. This initial path supports the global SQLite Supervisor. Plugin code is loaded only in child processes. + operationId: reload_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ReloadArtifactPluginRequest'} + responses: + "202": + description: Definition saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/enable: + post: + tags: [artifact-plugins] + summary: Enable an Artifact plugin + operationId: enable_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ChangeArtifactPluginStateRequest'} + responses: + "202": + description: Target saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/disable: + post: + tags: [artifact-plugins] + summary: Disable an Artifact plugin + operationId: disable_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ChangeArtifactPluginStateRequest'} + responses: + "202": + description: Target saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/unload: + post: + tags: [artifact-plugins] + summary: Unload an Artifact plugin + operationId: unload_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ChangeArtifactPluginStateRequest'} + responses: + "202": + description: Target saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} /v1/capabilities: get: tags: [capabilities] @@ -5999,6 +6160,10 @@ components: minLength: 1 maxLength: 128 pattern: '^[\x21-\x7E]+$' + schema_version: + type: string + nullable: true + description: Persisted schema version for a custom Artifact family. revision: type: integer minimum: 1 @@ -6113,6 +6278,10 @@ components: minLength: 1 maxLength: 128 pattern: '^[\x21-\x7E]+$' + schema_version: + type: string + nullable: true + description: Persisted schema version for a custom Artifact family. revision: type: integer minimum: 1 @@ -7049,12 +7218,76 @@ components: effect: type: string enum: [review_then_publish, review_then_publish_configuration, review_then_replace_tags] + CustomArtifactFamilySchema: + type: object + additionalProperties: false + required: [family, schema_version, content_schema, search_fields] + properties: + family: {type: string} + schema_version: {type: string} + content_schema: {type: object, additionalProperties: true} + search_fields: {type: array, items: {type: string}, description: Legacy persisted text fields; new plugins use search.} + search: {type: object, additionalProperties: true, description: Persistent declarative indexes and supported search modes.} + ReloadArtifactPluginRequest: + type: object + additionalProperties: false + required: [expected_generation] + properties: + expected_generation: {type: integer, minimum: 0} + ChangeArtifactPluginStateRequest: + type: object + additionalProperties: false + required: [expected_generation] + properties: + expected_generation: {type: integer, minimum: 1} + ArtifactPluginState: + type: object + additionalProperties: false + required: [binding_name, plugin_id, family, binding_generation, desired_state, phase, last_error] + properties: + binding_name: {type: string} + plugin_id: {type: string} + family: {type: string} + binding_generation: {type: integer, minimum: 1} + desired_state: {type: string, enum: [enabled, disabled, unloaded]} + phase: {type: string, enum: [activating, enabled, stopping, disabled, unloading, unloaded, failed]} + last_error: {type: string, nullable: true} + ArtifactPluginPage: + type: object + additionalProperties: false + required: [items] + properties: + items: {type: array, items: {$ref: '#/components/schemas/ArtifactPluginState'}} + EnableArtifactPluginResult: + type: object + additionalProperties: false + required: [binding_name, outcome, status_url, state, error] + properties: + binding_name: {type: string} + outcome: {type: string, enum: [accepted, skipped, failed]} + status_url: {type: string, description: Registration status URL for the asynchronous result.} + state: {$ref: '#/components/schemas/ArtifactPluginState'} + error: + type: string + nullable: true + enum: [conflict, invalid_plugin, unavailable, null] + description: Safe per-plugin failure code; null for accepted or skipped entries. + EnableAllArtifactPluginsResponse: + type: object + additionalProperties: false + required: [items] + properties: + items: {type: array, items: {$ref: '#/components/schemas/EnableArtifactPluginResult'}} Capabilities: type: object additionalProperties: false required: [source_types, artifact_families, memory_extraction, handoff_generation, search_modes, context_versions] properties: + custom_artifact_schemas: + type: array + default: [] + items: {$ref: '#/components/schemas/CustomArtifactFamilySchema'} prompts: type: object default: {} @@ -11061,13 +11294,36 @@ components: properties: method: {type: string, minLength: 1, pattern: '.*\S.*'} params: {type: object, additionalProperties: true, default: {}} + EmbeddingProfile: + type: object + additionalProperties: false + required: [profile_id, model, dimension] + description: Existing embedding space identity shared with Memory; no additional modality or version fields. + properties: + profile_id: {type: string, minLength: 1, pattern: '.*\S.*'} + model: {type: string, minLength: 1, pattern: '.*\S.*'} + dimension: {type: integer, minimum: 1} + distance: {type: string, enum: [l2], default: l2} + normalization: {type: string, enum: [none, unit], default: unit} SearchArtifactsRequest: type: object additionalProperties: false x-powercontext-artifact-search-validation: request - required: [query] properties: query: {type: string, minLength: 1, maxLength: 8192, pattern: '.*\S.*'} + query_vector: + type: object + additionalProperties: false + required: [profile, vector] + properties: + profile: {$ref: '#/components/schemas/EmbeddingProfile'} + vector: {type: array, minItems: 1, items: {type: number}} + indexes: + type: object + additionalProperties: false + properties: + text: {type: array, minItems: 1, uniqueItems: true, items: {type: string, minLength: 1}} + vector: {type: array, minItems: 1, uniqueItems: true, items: {type: string, minLength: 1}} limit: {type: integer, minimum: 1, maximum: 200, default: 10} mode: {type: string, minLength: 1, pattern: '.*\S.*'} filters: {type: object, additionalProperties: true} @@ -11966,7 +12222,9 @@ components: enum: [memory, atomic-memory, experience, skill, handoff, profile, prompt, topic-memory] ArtifactReadFamily: type: string - enum: [memory, atomic-memory, experience, skill, handoff, profile, prompt, topic-memory] + minLength: 1 + maxLength: 128 + pattern: '^[a-z][a-z0-9-]*$' StatsPeriod: type: string enum: [today, 7d, 30d] diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index 7448a04133..f6d9156e1f 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -14,6 +14,7 @@ """Stable public exports for PowerContext clients.""" +from powercontext import artifact_plugins from powercontext.artifacts import ( Artifact, ArtifactAddress, @@ -113,4 +114,5 @@ "SourceStore", "Sources", "Trigger", + "artifact_plugins", ] diff --git a/src/powercontext/artifact_plugins.py b/src/powercontext/artifact_plugins.py new file mode 100644 index 0000000000..e97634144a --- /dev/null +++ b/src/powercontext/artifact_plugins.py @@ -0,0 +1,140 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Declarations and generation contracts for Worker-loaded Artifact plugins.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from contextlib import AbstractAsyncContextManager, AsyncExitStack +from dataclasses import dataclass +from typing import TypeVar + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +from powercontext.artifacts import ArtifactLineage, ArtifactRef +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, FragmentProjection, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchOutcome +from powercontext.sources import SourceProjectionKey, SourceRef + +ResourceT = TypeVar("ResourceT") + + +@dataclass(frozen=True) +class ArtifactFamily: + family: str + schema_version: str + content_model: type[BaseModel] + search: ArtifactSearchDeclaration | None = None + + +@dataclass(frozen=True) +class InputContract: + projections: dict[str, SourceProjectionKey] + artifact_families: tuple[str, ...] = () + + +class GenerationSource(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + ref: SourceRef + projection: SourceProjectionKey + value: JsonValue + + +@dataclass(frozen=True) +class GenerationBatch: + sources: tuple[GenerationSource, ...] + + +class ArtifactDraft(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + family: str + schema_version: str + artifact_id: str = Field(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$") + expected_revision: int = Field(default=0, ge=0) + content: dict[str, JsonValue] + source_refs: tuple[SourceRef, ...] + artifact_refs: tuple[ArtifactRef, ...] = () + + +class ArtifactSnapshot(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + ref: ArtifactRef + lineage: ArtifactLineage + schema_version: str | None + content: dict[str, JsonValue] + source_refs: tuple[SourceRef, ...] = () + artifact_refs: tuple[ArtifactRef, ...] = () + + +class ArtifactReferencePage(BaseModel): + items: tuple[ArtifactRef, ...] + next_cursor: str | None = None + + +class GenerationContext: + """Scope-bound reads whose exact evidence can be used by generated drafts.""" + + def __init__( + self, + scope_id: str, + resources: AsyncExitStack, + read_artifact: Callable[[str, str, int | None], Awaitable[ArtifactSnapshot | None]], + *, + list_artifacts: Callable[..., Awaitable[ArtifactReferencePage]] | None = None, + search_artifacts: Callable[..., Awaitable[ArtifactSearchOutcome]] | None = None, + ) -> None: + self.scope_id = scope_id + self._resources = resources + self._read_artifact = read_artifact + self._list_artifacts = list_artifacts + self._search_artifacts = search_artifacts + + async def enter_async_context(self, resource: AbstractAsyncContextManager[ResourceT]) -> ResourceT: + return await self._resources.enter_async_context(resource) + + async def read_latest(self, family: str, artifact_id: str) -> ArtifactSnapshot | None: + """Read one current Artifact; separate reads do not establish a cross-family snapshot.""" + + return await self._read_artifact(family, artifact_id, None) + + async def read_revision(self, ref: ArtifactRef) -> ArtifactSnapshot: + """Read and register an exact revision, including a superseded historical revision.""" + + result = await self._read_artifact(ref.family, ref.artifact_id, ref.revision) + if result is None: + raise LookupError(ref) + return result + + async def list_artifacts(self, family: str, *, limit: int = 50, cursor: str | None = None) -> ArtifactReferencePage: + """Page visible current references; use a fixed manifest for historical enumeration.""" + if self._list_artifacts is None: + raise NotImplementedError("Artifact listing is unavailable") + return await self._list_artifacts(family, limit=limit, cursor=cursor) + + async def search_artifacts(self, family: str, query: str | None = None, **parameters) -> ArtifactSearchOutcome: + """Retrieve complete exact Artifacts through the shared search contract.""" + if self._search_artifacts is None: + raise NotImplementedError("Artifact search is unavailable") + return await self._search_artifacts(family, query, **parameters) + + +__all__ = [ + "ArtifactDraft", + "ArtifactFamily", + "ArtifactReferencePage", + "ArtifactSearchDeclaration", + "ArtifactSnapshot", + "FragmentProjection", + "GenerationBatch", + "GenerationContext", + "GenerationSource", + "InputContract", + "TextIndex", + "VectorIndex", +] diff --git a/src/powercontext/artifacts/indexes.py b/src/powercontext/artifacts/indexes.py new file mode 100644 index 0000000000..2be73ac3d6 --- /dev/null +++ b/src/powercontext/artifacts/indexes.py @@ -0,0 +1,168 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Declarative, persistent search projections for ordinary Artifact families.""" + +from __future__ import annotations + +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile + +Name = Annotated[str, Field(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")] +FieldName = Annotated[str, Field(min_length=1, max_length=128)] +Version = Annotated[str, Field(min_length=1, max_length=128, pattern=r".*\S.*")] +SearchMode = Literal["text", "vector", "hybrid"] + + +class _Declaration(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + +class FragmentProjection(_Declaration): + """Project an explicit persistent array; the framework does not split content.""" + + array_field: FieldName + id_field: FieldName + + +class TextIndex(_Declaration): + name: Name + kind: Literal["text"] = "text" + projection_version: Version + fields: tuple[FieldName, ...] = Field(min_length=1) + + @model_validator(mode="after") + def unique_fields(self): + if len(set(self.fields)) != len(self.fields): + raise ValueError("text fields must be distinct") # noqa: TRY003 + return self + + +class VectorIndex(_Declaration): + """Index persistent vectors or encode persistent text in one exact profile. + + For content vectors, profile_field names a persistent EmbeddingProfile object + alongside the vector. This prevents a producer replacing P1 with P2 in place. + Fragment projections resolve fields within each persistent array item. + """ + + name: Name + kind: Literal["vector"] = "vector" + projection_version: Version + source: Literal["text", "content"] + profile: EmbeddingProfile + fields: tuple[FieldName, ...] = () + vector_field: FieldName | None = None + profile_field: FieldName | None = None + fragments: FragmentProjection | None = None + + @model_validator(mode="after") + def coherent_source(self): + if not self.profile.profile_id.strip() or not self.profile.model.strip() or self.profile.dimension < 1: + raise ValueError("an explicit model, profile identity and positive dimension are required") # noqa: TRY003 + if self.source == "text": + if not self.fields or self.vector_field is not None or self.profile_field is not None: + raise ValueError("text vectors require text fields and cannot declare content vector fields") # noqa: TRY003 + if len(set(self.fields)) != len(self.fields): + raise ValueError("text fields must be distinct") # noqa: TRY003 + elif self.fields or self.vector_field is None or self.profile_field is None: + raise ValueError("content vectors require vector_field and profile_field, without text fields") # noqa: TRY003 + return self + + +SearchIndex = Annotated[TextIndex | VectorIndex, Field(discriminator="kind")] + + +class ArtifactSearchDeclaration(_Declaration): + version: Version + indexes: tuple[SearchIndex, ...] = Field(min_length=1) + modes: tuple[SearchMode, ...] = Field(min_length=1) + default_mode: SearchMode + defaults: dict[SearchMode, dict[Literal["text", "vector"], tuple[Name, ...]]] = Field(default_factory=dict) + candidate_limit: int = Field(default=200, ge=1, le=2000, strict=True) + rrf_k: int = Field(default=60, gt=0, strict=True) + text_weight: float = Field(default=1, ge=0) + vector_weight: float = Field(default=1, ge=0) + + @model_validator(mode="after") + def coherent_modes(self): + by_name = {index.name: index for index in self.indexes} + if len(by_name) != len(self.indexes) or len(set(self.modes)) != len(self.modes): + raise ValueError("index names and modes must be distinct") # noqa: TRY003 + if self.default_mode not in self.modes or set(self.defaults) - set(self.modes): + raise ValueError("defaults must refer to supported modes") # noqa: TRY003 + if self.text_weight + self.vector_weight <= 0: + raise ValueError("at least one fusion channel must have positive weight") # noqa: TRY003 + for mode in self.modes: + self.select_indexes(mode, self.defaults.get(mode)) + return self + + def select_indexes(self, mode, selection=None): # noqa: C901 - Keep the complete projection or publication invariant together. + if mode not in self.modes: + raise ValueError("search mode is not declared") # noqa: TRY003 + channels = {"text", "vector"} if mode == "hybrid" else {mode} + selection = self.defaults.get(mode) if selection is None else selection + if selection is None: + selection = {} + for channel in channels: + names = tuple(index.name for index in self.indexes if index.kind == channel) + if len(names) != 1: + raise ValueError("multiple indexes require explicit mode defaults") # noqa: TRY003 + selection[channel] = names + if set(selection) != channels: + raise ValueError("index selection must match the requested channels") # noqa: TRY003 + by_name = {index.name: index for index in self.indexes} + result = [] + for channel, names in selection.items(): + if not names or len(set(names)) != len(names): + raise ValueError("each channel requires distinct index names") # noqa: TRY003 + for name in names: + if name not in by_name or by_name[name].kind != channel: + raise ValueError("index is not declared for this channel") # noqa: TRY003 + result.append(by_name[name]) + profiles = [index.profile for index in result if isinstance(index, VectorIndex)] + if profiles and any(profile != profiles[0] for profile in profiles): + raise ValueError("selected vector indexes require the same complete EmbeddingProfile") # noqa: TRY003 + return tuple(result) + + +def validate_index_schema(index: TextIndex | VectorIndex, schema: dict[str, Any]) -> None: # noqa: C901 - Keep the complete projection or publication invariant together. + """Validate explicit required fields; no JSONPath or producer code is executed.""" + + def resolve(value): + seen = set() + while "$ref" in value: + ref = value["$ref"] + if not ref.startswith("#/$defs/") or ref in seen: + raise ValueError("index fields require local, nonrecursive schema definitions") # noqa: TRY003 + seen.add(ref) + value = schema["$defs"][ref.removeprefix("#/$defs/")] + return value + + def field(obj, name, kind): + obj = resolve(obj) + value = resolve(obj.get("properties", {}).get(name, {})) + if name not in obj.get("required", ()) or value.get("type") != kind: + raise ValueError(f"index field {name!r} must be a required {kind}") # noqa: TRY003 + return value + + obj = schema + if isinstance(index, VectorIndex) and index.fragments is not None: + array = field(obj, index.fragments.array_field, "array") + obj = resolve(array.get("items", {})) + if obj.get("type") != "object": + raise ValueError("fragments must be persistent objects") # noqa: TRY003 + field(obj, index.fragments.id_field, "string") + if isinstance(index, TextIndex) or index.source == "text": + for name in index.fields: + field(obj, name, "string") + else: + vector = field(obj, index.vector_field, "array") + if resolve(vector.get("items", {})).get("type") not in {"number", "integer"}: + raise ValueError("vectors must contain numbers") # noqa: TRY003 + field(obj, index.profile_field, "object") diff --git a/src/powercontext/artifacts/search.py b/src/powercontext/artifacts/search.py index a6e005f261..75d6471250 100644 --- a/src/powercontext/artifacts/search.py +++ b/src/powercontext/artifacts/search.py @@ -154,6 +154,15 @@ def __init__(self, family: str, *, field: str | None = None) -> None: super().__init__(f"Artifact search is unsupported for {family}" + (f": {field}" if field else "")) +class ArtifactSearchUnavailable(PowerContextError, RuntimeError): + """A declared search capability cannot serve a complete current projection.""" + + def __init__(self, family: str, reason: str) -> None: + self.family = family + self.reason = reason + super().__init__(f"Artifact search is unavailable for {family}: {reason}") + + class ArtifactSearchContractError(PowerContextError, RuntimeError): """A search implementation returned values that violate the public contract.""" @@ -179,6 +188,7 @@ def _require_finite_real(value: object, label: str) -> None: "ArtifactSearchMatch", "ArtifactSearchOutcome", "ArtifactSearchQuery", + "ArtifactSearchUnavailable", "ArtifactSearchUnsupported", "ArtifactSearcher", "ChannelScore", diff --git a/src/powercontext/builtin/artifact_plugin_validation.py b/src/powercontext/builtin/artifact_plugin_validation.py new file mode 100644 index 0000000000..dc5837f39c --- /dev/null +++ b/src/powercontext/builtin/artifact_plugin_validation.py @@ -0,0 +1,253 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +"""Isolated Artifact declaration validation without starting the generation runtime.""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib.util +import inspect +import json +import re +import sys +from datetime import timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Annotated, Any + +import yaml +from jsonschema import Draft202012Validator +from pydantic import BaseModel, ConfigDict, Field, JsonValue, SecretBytes, SecretStr, field_validator + +from powercontext.artifact_plugins import InputContract +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, validate_index_schema +from powercontext.sources import SourceProjectionKey, SourceRef + +_MAX_INPUT_BYTES = 1024 * 1024 +_MAX_OUTPUT_BYTES = 1024 * 1024 +_BUILTIN_FAMILIES = frozenset({ + "memory", + "atomic-memory", + "experience", + "skill", + "handoff", + "profile", + "prompt", + "topic-memory", +}) + + +class _ProcessingConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True, allow_inf_nan=False) + + interval_seconds: float = Field(gt=0) + max_workers: int = Field(ge=1) + timeout_seconds: float = Field(gt=0) + + @field_validator("interval_seconds") + @classmethod + def representable_interval(cls, value: float) -> float: + try: + interval = timedelta(seconds=value) + except OverflowError as error: + raise ValueError("interval_seconds exceeds the supported duration") from error # noqa: TRY003 + if interval.total_seconds() <= 0: + raise ValueError("interval_seconds must be at least one microsecond") # noqa: TRY003 + return value + + +class _PluginManifest(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + id: str = Field(pattern=r"^[a-z][a-z0-9-]{0,127}$") + version: str = Field(min_length=1, max_length=128, pattern=r".*\S.*") + entry: str = Field(min_length=3, max_length=4096) + binding_name: str = Field(pattern=r"^[a-z][a-z0-9-]{0,127}$") + scope_ids: list[Annotated[str, Field(min_length=1, max_length=256)]] | None = Field( + default=None, min_length=1, max_length=1000 + ) + max_sources: int = Field(ge=1) + processing: _ProcessingConfig + config: dict[str, JsonValue] + + @field_validator("scope_ids") + @classmethod + def unique_scopes(cls, value: list[str] | None) -> list[str] | None: + if value is not None and len(set(value)) != len(value): + raise ValueError("Scope IDs must be unique") # noqa: TRY003 + return None if value is None else sorted(value) + + +def read_manifest(directory: str) -> _PluginManifest: + path = Path(directory) / "plugin.yml" + if path.stat().st_size > _MAX_INPUT_BYTES: + raise ValueError("plugin manifest is too large") # noqa: TRY003 + return _PluginManifest.model_validate(yaml.safe_load(path.read_text(encoding="utf-8"))) + + +def code_fingerprint(directory: str) -> str: + root = Path(directory) + digest = hashlib.sha256() + files = sorted( + path + for path in root.rglob("*") + if path.is_file() and not {"__pycache__", ".venv", ".git"}.intersection(path.relative_to(root).parts) + ) + for path in files: + digest.update(str(path.relative_to(root)).encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def load_plugin(directory: str, manifest: _PluginManifest): + root = Path(directory) + entry, factory = manifest.entry.rsplit(":", 1) + path = (root / entry).resolve() + if Path(entry).is_absolute() or not path.is_relative_to(root.resolve()) or path.suffix != ".py": + raise ValueError("entry must be a Python file inside the plugin directory") # noqa: TRY003 + module_name = "powercontext_user_artifact_plugin" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ValueError("plugin entry is unavailable") # noqa: TRY003 + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + sys.path.insert(0, str(root)) + spec.loader.exec_module(module) + module.__dict__["create_pipeline"] = getattr(module, factory) + return module + + +def describe_plugin(directory: str, snapshot: dict[str, Any] | None = None) -> dict[str, Any]: + manifest = read_manifest(directory) if snapshot is None else _PluginManifest.model_validate(snapshot) + return describe_module(load_plugin(directory, manifest), manifest, directory) + + +def _input_declaration(inputs: InputContract) -> dict[str, Any]: + if not isinstance(inputs, InputContract) or not isinstance(inputs.projections, dict) or not inputs.projections: + raise ValueError("inputs must declare a nonempty Source projection mapping") # noqa: TRY003 + projections = {} + for name, projection in inputs.projections.items(): + SourceRef(source_type=name, source_id="validation") + projections[name] = SourceProjectionKey.model_validate(projection).model_dump(mode="json") + families = _read_families(inputs.artifact_families) + return {"projections": projections, **({"artifact_families": families} if families else {})} + + +def _read_families(families): + if not isinstance(families, tuple): + raise TypeError("read families must be a tuple") # noqa: TRY003 + for family in families: + if not isinstance(family, str) or re.fullmatch(r"[a-z][a-z0-9-]{0,127}", family) is None or family == "memory": + raise ValueError("read families must name canonical Artifact families") # noqa: TRY003 + if len(set(families)) != len(families): + raise ValueError("read families must be distinct") # noqa: TRY003 + return sorted(families) + + +def _configuration_json(value, serialized): + # Config stays private to management/Workers. Preserve credentials instead of + # persisting SecretStr's display mask, while retaining JSON normalization. + if isinstance(value, SecretStr): + return value.get_secret_value() + if isinstance(value, SecretBytes): + return value.get_secret_value().decode("utf-8") + if isinstance(value, dict) and isinstance(serialized, dict): + return {key: _configuration_json(value[key], item) for key, item in serialized.items()} + if isinstance(value, (list, tuple)) and isinstance(serialized, list): + return [_configuration_json(original, item) for original, item in zip(value, serialized, strict=True)] + return serialized + + +def _validate_factory(factory) -> None: + if not inspect.iscoroutinefunction(factory): + raise ValueError("plugin factory must be async") # noqa: TRY003 + try: + inspect.signature(factory).bind(None, None) + except (TypeError, ValueError) as error: + raise ValueError("plugin factory must accept context and configuration") from error # noqa: TRY003 + + +def describe_module(module, manifest: _PluginManifest, directory: str) -> dict[str, Any]: + if module.plugin_api_version != "1" or len(module.families) != 1: + raise ValueError("API version 1 requires exactly one output family") # noqa: TRY003 + _validate_factory(module.create_pipeline) + family = module.families[0] + if re.fullmatch(r"[a-z][a-z0-9-]{0,127}", family.family) is None or family.family in _BUILTIN_FAMILIES: + raise ValueError("a plugin needs a custom canonical family") # noqa: TRY003 + if not family.schema_version or len(family.schema_version) > 128: + raise ValueError("a bounded schema_version is required") # noqa: TRY003 + schema = family.content_model.model_json_schema() + Draft202012Validator.check_schema(schema) + if schema.get("type") != "object": + raise ValueError("Artifact content must be a JSON object") # noqa: TRY003 + fields = () + search = family.search + if search is not None: + search = ArtifactSearchDeclaration.model_validate(search) + for index in search.indexes: + validate_index_schema(index, schema) + # JSON validation restores strict Config types from the serialized definition snapshot. + validated_config = module.Config.model_validate_json(json.dumps(manifest.config, ensure_ascii=False)) + snapshot = manifest.model_dump(mode="json") + snapshot["config"] = _configuration_json( + validated_config.model_dump(mode="python", by_alias=True), + validated_config.model_dump(mode="json", by_alias=True), + ) + return { + "manifest": snapshot, + "declaration": { + "plugin_id": manifest.id, + "version": manifest.version, + "family": family.family, + "schema_version": family.schema_version, + "content_schema": schema, + "search_fields": list(fields), + **({"search": search.model_dump(mode="json")} if search is not None else {}), + **_input_declaration(module.inputs), + "fingerprint": code_fingerprint(directory), + }, + } + + +async def validate_plugin(directory: str, snapshot: dict[str, Any] | None = None) -> dict[str, Any]: + # Result files keep user import-time stdout out of the declaration protocol. + with TemporaryDirectory(prefix="powercontext-artifact-validation-") as temporary: + root = Path(temporary) + (root / "manifest.json").write_text(json.dumps(snapshot, ensure_ascii=False), encoding="utf-8") + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + __name__, + directory, + str(root / "manifest.json"), + str(root / "result.json"), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + async with asyncio.timeout(15): + await process.wait() + except BaseException: + if process.returncode is None: + process.kill() + await process.wait() + raise + if process.returncode: + raise ValueError("plugin declaration or configuration validation failed") # noqa: TRY003 + output = root / "result.json" + if output.stat().st_size > _MAX_OUTPUT_BYTES: + raise ValueError("plugin declaration is too large") # noqa: TRY003 + return json.loads(output.read_text()) + + +if __name__ == "__main__": + result = describe_plugin(sys.argv[1], json.loads(Path(sys.argv[2]).read_text())) + Path(sys.argv[3]).write_text(json.dumps(result, ensure_ascii=False)) diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 94ddb3abd0..cd4b6bddbc 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -34,6 +34,9 @@ if TYPE_CHECKING: import jieba +# Bump when document segmentation or token encoding changes; query stopwords do not affect projections. +TEXT_ANALYZER_VERSION = "jieba-0.42.1-words-v1" + _FTS_MIN_QUERY_COVERAGE = 0.25 _FTS_MIN_MATCHED_TERMS = 2 _FTS_SHORT_QUERY_MAX_TERMS = 2 @@ -418,6 +421,7 @@ def admits_fts_projection(query: str, projection: str, /, *, floor: AdmissionFlo __all__ = [ "DEFAULT_ADMISSION_FLOOR", + "TEXT_ANALYZER_VERSION", "AdmissionCounts", "AdmissionFloor", "EmptyFilters", diff --git a/src/powercontext/builtin/persistence/artifact_governance.py b/src/powercontext/builtin/persistence/artifact_governance.py index a4aadc36ba..8d5b11c594 100644 --- a/src/powercontext/builtin/persistence/artifact_governance.py +++ b/src/powercontext/builtin/persistence/artifact_governance.py @@ -17,6 +17,7 @@ from __future__ import annotations from enum import StrEnum +from typing import get_args from pydantic import BaseModel, ConfigDict, Field, model_validator from sqlalchemy import select, update @@ -25,6 +26,7 @@ from powercontext.artifacts import ArtifactRef from powercontext.builtin.persistence.errors import RepositoryNotFoundError, StoredPayloadConflictError from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE +from powercontext.builtin.records import BaseArtifactFamily from powercontext.errors import PowerContextError @@ -67,6 +69,30 @@ def validate_replacement(self) -> ArtifactGovernance: class ArtifactGovernanceRepository: """Read and transition governance state without changing the head Revision.""" + def __init__(self, *, prepared_indexes=None): + self.prepared_indexes = prepared_indexes or {} + + async def _prepare_custom_transition(self, connection, scope_id, family, artifact_id, lifecycle_state): + if family in get_args(BaseArtifactFamily): + return None + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher, lock_state + from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository + + await lock_state(connection, scope_id, family) + artifact = await custom_artifact_repository(family).latest(connection, scope_id, family, artifact_id) + publisher = ArtifactProjectionPublisher(None) + prepared = self.prepared_indexes.get((scope_id, family, artifact_id)) + if lifecycle_state is ArtifactLifecycleState.ACTIVE: + await publisher.validate_prepared( + connection, scope_id, family, artifact.content.schema_version, artifact.content.content, prepared + ) + return publisher, artifact, prepared + + async def _publish_custom_transition(self, connection, scope_id, custom): + if custom is not None: + publisher, artifact, prepared = custom + await publisher.publish(connection, scope_id, artifact, prepared) + async def get( self, connection: AsyncConnection, @@ -115,6 +141,7 @@ async def transition_merge( ) -> ArtifactGovernance: """Apply a service-validated group transition while retaining unchanged state generations.""" + custom = await self._prepare_custom_transition(connection, scope_id, family, artifact_id, lifecycle_state) _validate_transition(expected.lifecycle_state, lifecycle_state) current = await self.require_summary(connection, scope_id, family, artifact_id, expected) if ( @@ -147,6 +174,7 @@ async def transition_merge( ) if result.rowcount != 1: raise StoredPayloadConflictError("artifact-governance", (scope_id, family, artifact_id)) + await self._publish_custom_transition(connection, scope_id, custom) return requested async def transition( @@ -160,6 +188,7 @@ async def transition( replacement_artifact_id: str | None, /, ) -> ArtifactGovernance: + custom = await self._prepare_custom_transition(connection, scope_id, family, artifact_id, lifecycle_state) current = await self.get(connection, scope_id, family, artifact_id) if current.merged_into_id is not None: raise InvalidArtifactLifecycleError("frozen merge inputs require whole-group restoration") # noqa: TRY003 @@ -206,6 +235,7 @@ async def transition( raise StoredPayloadConflictError( "artifact-governance", (scope_id, family, artifact_id, expected_generation) ) + await self._publish_custom_transition(connection, scope_id, custom) return requested diff --git a/src/powercontext/builtin/persistence/artifact_indexes.py b/src/powercontext/builtin/persistence/artifact_indexes.py new file mode 100644 index 0000000000..f2d81bf163 --- /dev/null +++ b/src/powercontext/builtin/persistence/artifact_indexes.py @@ -0,0 +1,1009 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Transactional custom Artifact projections, build coordination and rebuilds.""" + +from __future__ import annotations + +import json +import math +import struct +from copy import deepcopy +from dataclasses import dataclass +from hashlib import sha256 +from typing import Any +from uuid import uuid4 + +from sqlalchemy import delete, insert, select, text, update + +from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchFamilyNotFound, ArtifactSearchUnavailable +from powercontext.builtin.artifacts.memory.canonical import canonical_embedding, canonical_json +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.artifacts.search import TEXT_ANALYZER_VERSION, analyze_text +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE as SCHEMAS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE as HEADS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_INDEX_CHANGES_TABLE as CHANGES, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_INDEX_COVERAGE_TABLE as COVERAGE, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_SEARCH_STATES_TABLE as STATES, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_SEARCH_TEXT_TABLE as TEXTS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTOR_INDEXES_TABLE as DEFINITIONS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTOR_SPACES_TABLE as SPACES, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTORS_TABLE as VECTORS, +) + +FORMAT = "float32-le-v1" + + +def identity_json(value): + # IDs, raw content and fragment locators must not be Unicode-normalized by hashing. + return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode() + + +def digest(value): + return sha256(identity_json(value)).hexdigest() + + +def search_declaration(durable): + if durable.get("search") is not None: + return ArtifactSearchDeclaration.model_validate(durable["search"]) + fields = durable.get("search_fields", []) + if fields: + return ArtifactSearchDeclaration( + version="legacy-fts-v1", + indexes=(TextIndex(name="lexical", projection_version="1", fields=tuple(fields)),), + modes=("text",), + default_mode="text", + ) + return None + + +def key(table, scope, family): + return (table.c.scope_id == scope) & (table.c.family == family) + + +async def lock_state(connection, scope, family): + values = { + "scope_id": scope, + "family": family, + "build_id": None, + "state": "pending", + "manifest": {}, + "cleanup_targets": [], + "start_seq": 0, + "applied_seq": 0, + "change_seq": 0, + } + if connection.dialect.name == "sqlite": + from sqlalchemy.dialects.sqlite import insert as dialect_insert + + stmt = dialect_insert(STATES).values(**values).on_conflict_do_nothing(index_elements=["scope_id", "family"]) + else: + from sqlalchemy.dialects.mysql import insert as dialect_insert + + stmt = dialect_insert(STATES).values(**values).on_duplicate_key_update(family=family) + await connection.execute(stmt) + # This is a real write on SQLite, and the same row lock as SELECT FOR UPDATE on MySQL. + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(change_seq=STATES.c.change_seq)) + return ( + (await connection.execute(select(STATES).where(key(STATES, scope, family)).with_for_update())).mappings().one() + ) + + +async def read_head(connection, scope, family, artifact_id, *, for_update=False): + statement = select(HEADS).where(key(HEADS, scope, family), HEADS.c.artifact_id == artifact_id) + if for_update: + statement = statement.with_for_update() + return (await connection.execute(statement)).mappings().one_or_none() + + +async def record_change(connection, scope, family, artifact_id, schema_version, operation): + state = await lock_state(connection, scope, family) + head = await read_head(connection, scope, family, artifact_id) + sequence = state["change_seq"] + 1 + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(change_seq=sequence)) + await connection.execute( + insert(CHANGES).values( + scope_id=scope, + family=family, + seq=sequence, + artifact_id=artifact_id, + revision=None if head is None else head["revision"], + schema_version=schema_version, + governance_generation=0 if head is None else head["governance_generation"], + operation=operation, + ) + ) + if state["state"] == "ready": + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(applied_seq=sequence)) + await prune_changes(connection, scope, family) + return sequence + + +async def prune_changes(connection, scope, family): + applied = await connection.scalar(select(STATES.c.applied_seq).where(key(STATES, scope, family))) + await connection.execute(delete(CHANGES).where(key(CHANGES, scope, family), CHANGES.c.seq <= applied)) + + +async def register_definition(connection, family, version, index): + spec = index.model_dump(mode="json") + if isinstance(index, TextIndex): + spec["analyzer_version"] = TEXT_ANALYZER_VERSION + space = None + if isinstance(index, VectorIndex): + fingerprint = sha256(canonical_json(index.profile.model_dump(mode="json"))).hexdigest() + backend = connection.dialect.name + space_row = ( + ( + await connection.execute( + select(SPACES).where( + SPACES.c.backend == backend, + SPACES.c.profile_fingerprint == fingerprint, + SPACES.c.storage_format_version == FORMAT, + ) + ) + ) + .mappings() + .first() + ) + if space_row is None: + space = uuid4().hex + await connection.execute( + insert(SPACES).values( + space_id=space, + profile_fingerprint=fingerprint, + profile_json=index.profile.model_dump(mode="json"), + dimension=index.profile.dimension, + distance=index.profile.distance, + normalization=index.profile.normalization, + storage_format_version=FORMAT, + backend=backend, + physical_table="pc_artifact_vectors" if backend == "sqlite" else f"pc_av_{space}", + provision_state="ready" if backend == "sqlite" else "declared", + ) + ) + else: + if space_row["profile_json"] != index.profile.model_dump(mode="json"): + raise ValueError("vector space fingerprint collision") # noqa: TRY003 + space = space_row["space_id"] + identity = { + "family": family, + "schema_version": version, + "index_name": index.name, + "projection_version": index.projection_version, + "space_id": space, + } + existing = ( + ( + await connection.execute( + select(DEFINITIONS) + .where(*(DEFINITIONS.c[name] == value for name, value in identity.items())) + .order_by(DEFINITIONS.c.definition_fingerprint) + ) + ) + .mappings() + .first() + ) + fingerprint = digest(spec) + exact = ( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == digest({**identity, "spec": spec})) + ) + ) + .mappings() + .first() + ) + if exact is not None: + if any(exact[name] != value for name, value in identity.items()) or exact["projection_spec"] != spec: + raise ValueError("index definition identity collision") # noqa: TRY003 + return dict(exact) + if existing is not None: + prior_spec = {k: v for k, v in existing["projection_spec"].items() if k != "analyzer_version"} + current_spec = {k: v for k, v in spec.items() if k != "analyzer_version"} + if prior_spec != current_spec: + raise ValueError("index projection version cannot be redefined") # noqa: TRY003 + if existing["definition_fingerprint"] == fingerprint: + return dict(existing) + row = dict( + index_def_id=digest({**identity, "spec": spec}), + **identity, + source_kind="text" if isinstance(index, TextIndex) else index.source, + projection_spec=spec, + definition_fingerprint=fingerprint, + ) + await connection.execute(insert(DEFINITIONS).values(**row)) + return row + + +def parse_index(definition): + spec = {k: v for k, v in definition["projection_spec"].items() if k != "analyzer_version"} + return (TextIndex if spec["kind"] == "text" else VectorIndex).model_validate(spec) + + +async def target_manifest(connection, scope, family, *, rebuild=False): # noqa: C901 - Initial build and repair share one fenced state transition. + """Initialize a fixed target or restart repair from its persisted declaration.""" + state = await lock_state(connection, scope, family) + manifest = deepcopy(state["manifest"]) + if manifest: + if not rebuild: + return manifest + for _, entry in entries(manifest): + entry.update(scan_cursor=None, scan_complete=False) + else: + rows = ( + ( + await connection.execute( + select(SCHEMAS).where(SCHEMAS.c.family == family).order_by(SCHEMAS.c.schema_version) + ) + ) + .mappings() + .all() + ) + if not rows: + raise ArtifactSearchFamilyNotFound(family) + manifest = {"schemas": {}, "config": None, "default_schema": None} + for row in rows: + declaration = search_declaration(row["declaration"]) + if declaration is None: + continue + schema = row["schema_version"] + if manifest["config"] is None: + manifest["config"] = declaration.model_dump(mode="json") + manifest["default_schema"] = schema + indexes = {} + for index in declaration.indexes: + definition = await register_definition(connection, family, schema, index) + indexes[index.name] = { + "index_def_id": definition["index_def_id"], + "space_id": definition["space_id"], + "scan_cursor": None, + "scan_complete": False, + } + manifest["schemas"][schema] = {"indexes": indexes} + targets = list(state["cleanup_targets"]) + if state["build_id"] is not None: + for _, entry in entries(state["manifest"]): + target = { + "build_id": state["build_id"], + "index_def_id": entry["index_def_id"], + "space_id": entry["space_id"], + } + if target not in targets: + targets.append(target) + await connection.execute( + update(STATES) + .where(key(STATES, scope, family)) + .values( + build_id=uuid4().hex, + state="cleaning" if targets else "building", + manifest=manifest, + cleanup_targets=targets, + validation_token=None, + start_seq=state["change_seq"], + applied_seq=state["change_seq"], + ) + ) + await prune_changes(connection, scope, family) + return manifest + + +def entries(manifest): + return tuple( + (schema, entry) + for schema, config in manifest.get("schemas", {}).items() + for entry in config["indexes"].values() + ) + + +def vector_bytes(values, profile): + if any(isinstance(v, bool) or not isinstance(v, (int, float)) for v in values): + raise ValueError("vectors must contain finite numbers") # noqa: TRY003 + vector = canonical_embedding(values, dimension=profile.dimension, normalization=profile.normalization) + try: + encoded = struct.pack(f"<{len(vector)}f", *vector) + except (OverflowError, struct.error) as error: + raise ValueError("vector exceeds float32 range") from error # noqa: TRY003 + if not all(math.isfinite(v) for v in struct.unpack(f"<{len(vector)}f", encoded)): + raise ValueError("vector exceeds float32 range") # noqa: TRY003 + return encoded + + +@dataclass(frozen=True) +class PreparedIndex: + definition: dict[str, Any] + fragments: tuple[dict[str, Any], ...] + input_hash: str + + +@dataclass(frozen=True) +class PreparedArtifactIndexes: + build_id: str | None + content_hash: str + projections: dict[str, PreparedIndex] + + +class ArtifactProjectionPublisher: + """Prepare without a transaction; publish only under the build coordinator.""" + + def __init__(self, database, embedding_model=None): + self.database = database + self.embedding_model = embedding_model + + async def prepare_index(self, definition, content): # noqa: C901 - Keep the complete projection or publication invariant together. + index = parse_index(definition) + if isinstance(index, VectorIndex) and index.fragments is not None: + parts = content[index.fragments.array_field] + if not isinstance(parts, list): + raise ValueError("fragment projection requires a persistent array") # noqa: TRY003 + else: + parts = [content] + fragments = [] + texts = [] + seen = set() + for offset, part in enumerate(parts): + fragment_id = "__whole__" + locator = {"fields": list(index.fields)} + if isinstance(index, VectorIndex) and index.fragments is not None: + fragment_id = part[index.fragments.id_field] + if not isinstance(fragment_id, str) or not 1 <= len(fragment_id) <= 128 or fragment_id == "__whole__": + raise ValueError("fragment identity must be a bounded persistent string") # noqa: TRY003 + locator = {"array": index.fragments.array_field, "id": fragment_id, "offset": offset} + if fragment_id in seen: + raise ValueError("fragment identities must be unique") # noqa: TRY003 + seen.add(fragment_id) + value = {"fragment_id": fragment_id, "fragment_locator": identity_json(locator).decode()} + if isinstance(index, TextIndex) or index.source == "text": + fields = [part[field] for field in index.fields] + if any(not isinstance(field, str) for field in fields): + raise ValueError("text projection requires strings") # noqa: TRY003 + original = "\n".join(fields) + if isinstance(index, TextIndex): + value["searchable_text"] = analyze_text(original) + else: + texts.append(original) + else: + if EmbeddingProfile.model_validate(part[index.profile_field]) != index.profile: + raise ValueError("persistent vector profile differs from the declared index") # noqa: TRY003 + value["embedding"] = vector_bytes(part[index.vector_field], index.profile) + value["input_hash"] = digest({"projection": definition["definition_fingerprint"], "part": part}) + fragments.append(value) + if texts: + model = self.embedding_model + if model is None or model.profile != index.profile: + raise ArtifactSearchUnavailable( + definition["family"], "embedding resource unavailable for declared profile" + ) + result = await model.embed(tuple(texts)) + if len(result.vectors) != len(texts): + raise ValueError("embedding result must match the prepared input count") # noqa: TRY003 + for fragment, vector in zip(fragments, result.vectors, strict=True): + fragment["embedding"] = vector_bytes(vector, index.profile) + return PreparedIndex( + definition, + tuple(fragments), + digest({"projection": definition["definition_fingerprint"], "content": content}), + ) + + async def prepare(self, scope, family, schema_version, content): + content = deepcopy(content) + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().first() + if state is None or state["state"] != "ready": + return PreparedArtifactIndexes(None, digest(content), {}) + definitions = {} + for schema, entry in entries(state["manifest"]): + if schema == schema_version: + definitions[entry["index_def_id"]] = dict( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == entry["index_def_id"]) + ) + ) + .mappings() + .one() + ) + projections = {} + try: + for identity, definition in definitions.items(): + projections[identity] = await self.prepare_index(definition, content) + except Exception: + # A rebuild may have suspended this index while encoding was in flight. + # Content writes then require only the durable change log. + async with self.database.transaction(consistent_snapshot=True) as connection: + current = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if current["state"] == "ready": + raise + return PreparedArtifactIndexes(None, digest(content), {}) + return PreparedArtifactIndexes(state["build_id"], digest(content), projections) + + async def validate_prepared(self, connection, scope, family, schema_version, content, prepared): + state = await lock_state(connection, scope, family) + if state["state"] != "ready": + return state + expected = {entry["index_def_id"] for schema, entry in entries(state["manifest"]) if schema == schema_version} + if prepared is None: + if expected: + raise ValueError("ready Artifact indexes must be prepared before the write transaction") # noqa: TRY003 + elif ( + prepared.build_id != state["build_id"] + or prepared.content_hash != digest(content) + or set(prepared.projections) != expected + ): + raise ValueError("Artifact content or search build changed; prepare again before committing") # noqa: TRY003 + return state + + async def publish(self, connection, scope, artifact, prepared): + family, identity = artifact.family, artifact.artifact_id + state = await lock_state(connection, scope, family) + head = await read_head(connection, scope, family, identity, for_update=True) + if state["state"] == "ready": + if head["lifecycle_state"] == "active": + await self.validate_prepared( + connection, scope, family, artifact.content.schema_version, artifact.content.content, prepared + ) + for _, entry in entries(state["manifest"]): + projection = ( + None + if prepared is None or head["lifecycle_state"] != "active" + else prepared.projections.get(entry["index_def_id"]) + ) + await self.replace( + connection, + scope, + family, + identity, + artifact.revision, + head["governance_generation"], + state["build_id"], + entry["index_def_id"], + projection, + ) + await record_change(connection, scope, family, identity, artifact.content.schema_version, "upsert") + + async def replace( + self, connection, scope, family, artifact_id, revision, governance, build_id, index_def_id, prepared + ): + state = await lock_state(connection, scope, family) + if ( + state["build_id"] != build_id + or state["state"] not in {"ready", "building"} + or state["validation_token"] is not None + ): + raise ValueError("projection build is not writable") # noqa: TRY003 + if index_def_id not in {entry["index_def_id"] for _, entry in entries(state["manifest"])}: + raise ValueError("projection definition is not in the current build") # noqa: TRY003 + definition = dict( + (await connection.execute(select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == index_def_id))) + .mappings() + .one() + ) + index = parse_index(definition) + table = await projection_table(connection, definition) + await connection.execute( + delete(table).where( + projection_key(table, scope, family, build_id, index_def_id), table.c.artifact_id == artifact_id + ) + ) + await connection.execute( + delete(COVERAGE).where( + projection_key(COVERAGE, scope, family, build_id, index_def_id), COVERAGE.c.artifact_id == artifact_id + ) + ) + if prepared is None: + return + if prepared.definition != definition: + raise ValueError("prepared definition does not match its index") # noqa: TRY003 + schema = definition["schema_version"] + rows = [] + for fragment in prepared.fragments: + identity = ( + scope, + family, + artifact_id, + revision, + schema, + definition["index_def_id"], + build_id, + fragment["fragment_id"], + ) + row = dict( + scope_id=scope, + family=family, + artifact_id=artifact_id, + revision=revision, + schema_version=schema, + index_def_id=definition["index_def_id"], + build_id=build_id, + governance_generation=governance, + **fragment, + ) + if isinstance(index, VectorIndex): + row.update( + vector_id=digest(identity), space_id=definition["space_id"], dimension=index.profile.dimension + ) + if connection.dialect.name != "sqlite": + row["embedding"] = list(struct.unpack(f"<{index.profile.dimension}f", row["embedding"])) + else: + row["text_id"] = digest(identity) + rows.append(row) + if rows: + await connection.execute(insert(table), rows) + await connection.execute( + insert(COVERAGE).values( + scope_id=scope, + family=family, + index_def_id=index_def_id, + build_id=build_id, + artifact_id=artifact_id, + revision=revision, + schema_version=schema, + fragment_count=len(rows), + projection_input_hash=prepared.input_hash, + governance_generation=governance, + ) + ) + + +def projection_key(table, scope, family, build_id, index_def_id): + return key(table, scope, family) & (table.c.build_id == build_id) & (table.c.index_def_id == index_def_id) + + +async def projection_table(connection, definition): + if definition["space_id"] is None: + return TEXTS + space = ( + (await connection.execute(select(SPACES).where(SPACES.c.space_id == definition["space_id"]))).mappings().one() + ) + if space["provision_state"] != "ready": + raise ArtifactSearchUnavailable(definition["family"], "vector space requires database initialization") + if connection.dialect.name == "sqlite": + return VECTORS + from powercontext.builtin.persistence.artifact_vector_space import native_vector_table + + try: + return native_vector_table(space) + except ValueError as error: + raise ArtifactSearchUnavailable(definition["family"], "registered vector space is incompatible") from error + + +class ArtifactIndexBuilder: + """Resume cleanup, scans and replay while the complete family search is unavailable.""" + + def __init__(self, database, embedding_model=None): + self.database = database + self.publisher = ArtifactProjectionPublisher(database, embedding_model) + + async def refresh(self, scope, family, *, rebuild=False): + if not rebuild: + async with self.database.transaction(consistent_snapshot=True) as connection: + manifest = await connection.scalar(select(STATES.c.manifest).where(key(STATES, scope, family))) + if manifest: + return manifest + async with self.database.transaction() as connection: + return await target_manifest(connection, scope, family, rebuild=rebuild) + + async def clean(self, scope, family, build_id, *, limit=1000): + """Delete a bounded batch, fenced by the current build and its coordinator lock.""" + async with self.database.transaction() as connection: + state = await lock_state(connection, scope, family) + if state["build_id"] != build_id or state["state"] != "cleaning": + return False + targets = list(state["cleanup_targets"]) + remaining = limit + while targets and remaining: + target = targets[0] + definition = ( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == target["index_def_id"]) + ) + ) + .mappings() + .one() + ) + tables = [COVERAGE] + # An unprovisioned space has never accepted projection rows. + if ( + target["space_id"] is None + or await connection.scalar( + select(SPACES.c.provision_state).where(SPACES.c.space_id == target["space_id"]) + ) + == "ready" + ): + tables.insert(0, await projection_table(connection, definition)) + complete = True + for table in tables: + where = projection_key(table, scope, family, target["build_id"], target["index_def_id"]) + primary = table.c.artifact_id if table is COVERAGE else next(iter(table.primary_key.columns)) + ids = (await connection.execute(select(primary).where(where).limit(remaining))).scalars().all() + if ids: + await connection.execute(delete(table).where(where, primary.in_(ids))) + remaining -= len(ids) + if not remaining: + complete = False + break + if complete: + targets.pop(0) + await connection.execute( + update(STATES) + .where(key(STATES, scope, family)) + .values(cleanup_targets=targets, state="cleaning" if targets else "building") + ) + return not targets + + async def build(self, scope, family): # noqa: C901 - Resume ordered cleanup, scans, replay and validation. + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if state["state"] == "ready": + return True + if not state["manifest"]: + return False + build_id = state["build_id"] + if state["state"] == "cleaning" and not await self.clean(scope, family, build_id): + return False + # Reclaim an interrupted validator. Any previous validator must still present + # its own token at commit, so it cannot publish after this recovery. + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if current["build_id"] != build_id or current["state"] != "building": + return False + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(validation_token=None)) + manifest = current["manifest"] + for _, entry in entries(manifest): + if entry["scan_complete"]: + continue + cursor = entry["scan_cursor"] or "" + while True: + async with self.database.transaction(consistent_snapshot=True) as connection: + identities = ( + ( + await connection.execute( + select(HEADS.c.artifact_id) + .where(key(HEADS, scope, family), HEADS.c.artifact_id > cursor) + .order_by(HEADS.c.artifact_id) + .limit(100) + ) + ) + .scalars() + .all() + ) + if not identities: + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if not self._writable(current, build_id): + return False + progress = deepcopy(current["manifest"]) + for _, item in entries(progress): + if item["index_def_id"] == entry["index_def_id"]: + item["scan_complete"] = True + await connection.execute( + update(STATES).where(key(STATES, scope, family)).values(manifest=progress) + ) + break + for identity in identities: + if not await self._project_current( + scope, family, build_id, entry["index_def_id"], identity, cursor=identity + ): + return False + cursor = identity + for _ in range(8): + if not await self._catch_up(scope, family, build_id): + return False + if await self._validate_and_publish(scope, family, build_id): + return True + return False + + @staticmethod + def _writable(state, build_id): + return state["build_id"] == build_id and state["state"] == "building" and state["validation_token"] is None + + async def _project_current(self, scope, family, build_id, index_def_id, identity, *, cursor=None): + from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository + + repository = custom_artifact_repository(family) + for _ in range(8): + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if not self._writable(state, build_id): + return False + definition = dict( + (await connection.execute(select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == index_def_id))) + .mappings() + .one() + ) + head = await read_head(connection, scope, family, identity) + artifact = None + if head is not None and head["lifecycle_state"] == "active": + artifact = await repository.get( + connection, scope, ArtifactRef(family=family, artifact_id=identity, revision=head["revision"]) + ) + prepared = None + if artifact is not None and artifact.content.schema_version == definition["schema_version"]: + prepared = await self.publisher.prepare_index(definition, artifact.content.content) + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if not self._writable(current, build_id): + return False + if await read_head(connection, scope, family, identity, for_update=True) != head: + continue + await self.publisher.replace( + connection, + scope, + family, + identity, + 0 if head is None else head["revision"], + 0 if head is None else head["governance_generation"], + build_id, + index_def_id, + prepared, + ) + if cursor is not None: + progress = deepcopy(current["manifest"]) + for _, entry in entries(progress): + if entry["index_def_id"] == index_def_id: + entry["scan_cursor"] = max(cursor, entry["scan_cursor"] or "") + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(manifest=progress)) + return True + return False + + async def _catch_up(self, scope, family, build_id): + while True: + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if not self._writable(state, build_id): + return False + changes = ( + ( + await connection.execute( + select(CHANGES) + .where(key(CHANGES, scope, family), CHANGES.c.seq > state["applied_seq"]) + .order_by(CHANGES.c.seq) + .limit(100) + ) + ) + .mappings() + .all() + ) + if not changes: + return True + for change in changes: + for _, entry in entries(state["manifest"]): + if not await self._project_current( + scope, family, build_id, entry["index_def_id"], change["artifact_id"] + ): + return False + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if not self._writable(current, build_id): + return False + await connection.execute( + update(STATES) + .where(key(STATES, scope, family)) + .values(applied_seq=max(change["seq"], current["applied_seq"])) + ) + await prune_changes(connection, scope, family) + + async def _validate_and_publish(self, scope, family, build_id): + token = uuid4().hex + async with self.database.transaction() as connection: + state = await lock_state(connection, scope, family) + if not self._writable(state, build_id) or state["applied_seq"] != state["change_seq"]: + return False + if not all(entry["scan_complete"] for _, entry in entries(state["manifest"])): + return False + manifest, sequence = state["manifest"], state["change_seq"] + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(validation_token=token)) + try: + async with self.database.transaction(consistent_snapshot=True) as connection: + valid = await self._coverage_valid(connection, scope, family, build_id, manifest) + if not valid: + return False + async with self.database.transaction() as connection: + state = await lock_state(connection, scope, family) + if ( + state["build_id"] != build_id + or state["state"] != "building" + or state["validation_token"] != token + or state["change_seq"] != sequence + or state["manifest"] != manifest + ): + return False + await connection.execute( + update(STATES).where(key(STATES, scope, family)).values(state="ready", validation_token=None) + ) + await prune_changes(connection, scope, family) + return True + finally: + async with self.database.transaction() as connection: + await lock_state(connection, scope, family) + await connection.execute( + update(STATES) + .where( + key(STATES, scope, family), STATES.c.build_id == build_id, STATES.c.validation_token == token + ) + .values(validation_token=None) + ) + + async def _coverage_valid(self, connection, scope, family, build_id, manifest): # noqa: C901 - Keep the complete projection or publication invariant together. + from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository + + repository = custom_artifact_repository(family) + heads = ( + ( + await connection.execute( + select(HEADS).where(key(HEADS, scope, family), HEADS.c.lifecycle_state == "active") + ) + ) + .mappings() + .all() + ) + artifacts = {} + for head in heads: + artifact = await repository.get( + connection, + scope, + ArtifactRef(family=family, artifact_id=head["artifact_id"], revision=head["revision"]), + ) + artifacts[head["artifact_id"]] = (head, artifact) + for schema, entry in entries(manifest): + definition = dict( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == entry["index_def_id"]) + ) + ) + .mappings() + .one() + ) + table = await projection_table(connection, definition) + index = parse_index(definition) + if isinstance(index, VectorIndex): + probe = vector_bytes([1.0] * index.profile.dimension, index.profile) + if connection.dialect.name == "sqlite": + await connection.execute(text("SELECT vec_distance_l2(:vector, :vector)"), {"vector": probe}) + else: + from sqlalchemy import bindparam + + values = list(struct.unpack(f"<{index.profile.dimension}f", probe)) + await connection.execute( + text("SELECT l2_distance(:probe, :probe)").bindparams( + bindparam("probe", type_=table.c.embedding.type) + ), + {"probe": values}, + ) + coverage = { + row["artifact_id"]: row + for row in ( + await connection.execute( + select(COVERAGE).where(projection_key(COVERAGE, scope, family, build_id, entry["index_def_id"])) + ) + ).mappings() + } + expected = { + identity for identity, (_, artifact) in artifacts.items() if artifact.content.schema_version == schema + } + if set(coverage) != expected: + return False + rows = ( + ( + await connection.execute( + select(table).where(projection_key(table, scope, family, build_id, entry["index_def_id"])) + ) + ) + .mappings() + .all() + ) + by_artifact = {} + for row in rows: + if row["artifact_id"] not in expected: + return False + by_artifact.setdefault(row["artifact_id"], []).append(row) + for identity in expected: + head, artifact = artifacts[identity] + content = artifact.content.content + row = coverage[identity] + parts = ( + content[index.fragments.array_field] + if isinstance(index, VectorIndex) and index.fragments + else [content] + ) + if ( + row["revision"], + row["schema_version"], + row["governance_generation"], + row["fragment_count"], + row["projection_input_hash"], + ) != ( + head["revision"], + schema, + head["governance_generation"], + len(parts), + digest({"projection": definition["definition_fingerprint"], "content": content}), + ): + return False + projected = by_artifact.get(identity, []) + if len(projected) != len(parts): + return False + expected_parts = { + ( + part[index.fragments.id_field] + if isinstance(index, VectorIndex) and index.fragments + else "__whole__" + ): (offset, part) + for offset, part in enumerate(parts) + } + if {item["fragment_id"] for item in projected} != set(expected_parts): + return False + for item in projected: + offset, part = expected_parts[item["fragment_id"]] + locator = ( + {"array": index.fragments.array_field, "id": item["fragment_id"], "offset": offset} + if isinstance(index, VectorIndex) and index.fragments + else {"fields": list(index.fields)} + ) + if ( + item["input_hash"] != digest({"projection": definition["definition_fingerprint"], "part": part}) + or item["fragment_locator"] != identity_json(locator).decode() + ): + return False + if isinstance(index, TextIndex): + if item["searchable_text"] != analyze_text("\n".join(part[field] for field in index.fields)): + return False + else: + if (item["space_id"], item["dimension"]) != (definition["space_id"], index.profile.dimension): + return False + vector = ( + item["embedding"] + if connection.dialect.name == "sqlite" + else struct.pack(f"<{index.profile.dimension}f", *item["embedding"]) + ) + if len(vector) != 4 * index.profile.dimension or not all( + math.isfinite(value) for value in struct.unpack(f"<{index.profile.dimension}f", vector) + ): + return False + if index.source == "content" and vector != vector_bytes( + part[index.vector_field], index.profile + ): + return False + if any( + ( + item["scope_id"], + item["family"], + item["revision"], + item["schema_version"], + item["governance_generation"], + item["index_def_id"], + ) + != ( + scope, + family, + head["revision"], + schema, + head["governance_generation"], + definition["index_def_id"], + ) + for item in projected + ): + return False + return True diff --git a/src/powercontext/builtin/persistence/artifact_search_schema.py b/src/powercontext/builtin/persistence/artifact_search_schema.py new file mode 100644 index 0000000000..1ebf94bc86 --- /dev/null +++ b/src/powercontext/builtin/persistence/artifact_search_schema.py @@ -0,0 +1,239 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Shared search build state and exact, index-isolated projections.""" + +from sqlalchemy import ( + JSON, + BigInteger, + CheckConstraint, + Column, + ForeignKeyConstraint, + Index, + Integer, + LargeBinary, + Table, + Text, + UniqueConstraint, +) + + +def artifact_search_tables(metadata, identity): + spaces = Table( + "pc_artifact_vector_spaces", + metadata, + Column("space_id", identity(32), primary_key=True), + Column("profile_fingerprint", identity(64), nullable=False), + Column("profile_json", JSON, nullable=False), + Column("dimension", Integer, nullable=False), + Column("distance", identity(16), nullable=False), + Column("normalization", identity(16), nullable=False), + Column("storage_format_version", identity(64), nullable=False), + Column("backend", identity(32), nullable=False), + Column("physical_table", identity(128), nullable=False), + Column("provision_state", identity(32), nullable=False), + UniqueConstraint( + "backend", "profile_fingerprint", "storage_format_version", name="uq_pc_artifact_vector_space" + ), + CheckConstraint("dimension > 0", name="ck_pc_artifact_vector_dimension"), + ) + definitions = Table( + "pc_artifact_vector_indexes", + metadata, + Column("index_def_id", identity(64), primary_key=True), + Column("family", identity(128), nullable=False), + Column("index_name", identity(128), nullable=False), + Column("schema_version", identity(128), nullable=False), + Column("projection_version", identity(128), nullable=False), + Column("space_id", identity(32)), + Column("source_kind", identity(16), nullable=False), + Column("projection_spec", JSON, nullable=False), + Column("definition_fingerprint", identity(64), nullable=False), + UniqueConstraint( + "family", + "index_name", + "schema_version", + "projection_version", + "space_id", + "definition_fingerprint", + name="uq_pc_artifact_index_definition", + ), + ForeignKeyConstraint(("space_id",), ("pc_artifact_vector_spaces.space_id",)), + ForeignKeyConstraint( + ("family", "schema_version"), + ("pc_artifact_family_schemas.family", "pc_artifact_family_schemas.schema_version"), + ), + ) + states = Table( + "pc_artifact_search_states", + metadata, + Column("scope_id", identity(256), primary_key=True), + Column("family", identity(128), primary_key=True), + Column("build_id", identity(32)), + Column("state", identity(16), nullable=False), + Column("manifest", JSON, nullable=False), + Column("cleanup_targets", JSON, nullable=False), + Column("validation_token", identity(32)), + Column("start_seq", BigInteger, nullable=False, server_default="0"), + Column("applied_seq", BigInteger, nullable=False, server_default="0"), + Column("change_seq", BigInteger, nullable=False, server_default="0"), + ) + coverage = Table( + "pc_artifact_index_coverage", + metadata, + Column("scope_id", identity(256), primary_key=True), + Column("family", identity(128), primary_key=True), + Column("build_id", identity(32), primary_key=True), + Column("index_def_id", identity(64), primary_key=True), + Column("artifact_id", identity(128), primary_key=True), + Column("revision", Integer, nullable=False), + Column("schema_version", identity(128), nullable=False), + Column("fragment_count", Integer, nullable=False), + Column("projection_input_hash", identity(64), nullable=False), + Column("governance_generation", BigInteger, nullable=False), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + CheckConstraint("fragment_count >= 0", name="ck_pc_artifact_index_fragment_count"), + ) + changes = Table( + "pc_artifact_index_changes", + metadata, + Column("scope_id", identity(256), primary_key=True), + Column("family", identity(128), primary_key=True), + Column("seq", BigInteger, primary_key=True), + Column("artifact_id", identity(128), nullable=False), + Column("revision", Integer), + Column("operation", identity(16), nullable=False), + Column("schema_version", identity(128)), + Column("governance_generation", BigInteger, nullable=False), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + ) + + def rows(name, vector): + return Table( + name, + metadata, + Column("vector_id" if vector else "text_id", identity(64), primary_key=True), + Column("scope_id", identity(256), nullable=False), + Column("family", identity(128), nullable=False), + Column("artifact_id", identity(128), nullable=False), + Column("revision", Integer, nullable=False), + Column("schema_version", identity(128), nullable=False), + Column("index_def_id", identity(64), nullable=False), + Column("build_id", identity(32), nullable=False), + Column("fragment_id", identity(128), nullable=False), + Column("fragment_locator", Text, nullable=False), + Column("input_hash", identity(64), nullable=False), + Column("governance_generation", BigInteger, nullable=False), + *( + ( + Column("space_id", identity(32), nullable=False), + Column("dimension", Integer, nullable=False), + Column("embedding", LargeBinary, nullable=False), + ) + if vector + else (Column("searchable_text", Text, nullable=False),) + ), + UniqueConstraint("build_id", "index_def_id", "artifact_id", "fragment_id", name=f"uq_{name}_fragment"), + ForeignKeyConstraint( + ("scope_id", "family", "artifact_id", "revision"), + ("pc_artifacts.scope_id", "pc_artifacts.family", "pc_artifacts.artifact_id", "pc_artifacts.revision"), + ), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + Index( + f"ix_{name}_search", + "scope_id", + "family", + "build_id", + *(("space_id",) if vector else ()), + "index_def_id", + "artifact_id", + ), + *( + ( + CheckConstraint( + "dimension > 0 AND length(embedding) = 4 * dimension", name="ck_pc_artifact_vectors_encoding" + ), + ) + if vector + else () + ), + ) + + return ( + spaces, + definitions, + states, + coverage, + changes, + rows("pc_artifact_vectors", True), + rows("pc_artifact_search_text", False), + ) + + +def ensure_search_support(connection): # noqa: C901 - Validate the fixed layout before repairing derived FTS support. + """Create missing backend full-text support during shared database initialization.""" + from sqlalchemy import inspect, text + + from powercontext.builtin.persistence.tables import ARTIFACT_SEARCH_TABLES + + inspector = inspect(connection) + for table in ARTIFACT_SEARCH_TABLES: + if table.name == "pc_artifact_vectors" and connection.dialect.name != "sqlite": + continue + name, expected = table.name, set(table.c.keys()) + if {column["name"] for column in inspector.get_columns(name)} != expected: + raise ValueError("Artifact search schema is incompatible; a database migration is required") # noqa: TRY003 + if inspector.has_table("pc_artifact_index_generations"): + raise ValueError("Legacy Artifact search generations require an explicit database migration") # noqa: TRY003 + if connection.dialect.name == "sqlite" and inspector.has_table("pc_artifact_search_fts"): + columns = {column["name"] for column in inspector.get_columns("pc_artifact_search_fts")} + if not {"build_id", "index_def_id"}.issubset(columns): + raise ValueError("Artifact full-text schema is incompatible; a database migration is required") # noqa: TRY003 + if connection.dialect.name != "sqlite": + indexes = {index["name"] for index in inspect(connection).get_indexes("pc_artifact_search_text")} + if "ix_pc_artifact_search_fts" not in indexes: + connection.exec_driver_sql( + "CREATE FULLTEXT INDEX ix_pc_artifact_search_fts " + "ON pc_artifact_search_text (searchable_text) WITH PARSER SPACE" + ) + return + helpers = ( + ( + "pc_artifact_search_fts", + "CREATE VIRTUAL TABLE pc_artifact_search_fts USING fts5(text_id UNINDEXED, scope_id UNINDEXED, family UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED, build_id UNINDEXED, index_def_id UNINDEXED, searchable_text, tokenize='unicode61')", + ), + ( + "pc_artifact_search_fts_insert", + "CREATE TRIGGER pc_artifact_search_fts_insert AFTER insert ON pc_artifact_search_text BEGIN INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END", + ), + ( + "pc_artifact_search_fts_update", + "CREATE TRIGGER pc_artifact_search_fts_update AFTER update ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id;INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END", + ), + ( + "pc_artifact_search_fts_delete", + "CREATE TRIGGER pc_artifact_search_fts_delete AFTER delete ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id; END", + ), + ) + missing = False + for name, ddl in helpers: + if connection.scalar(text("SELECT 1 FROM sqlite_master WHERE name=:name"), {"name": name}) is None: + connection.exec_driver_sql(ddl) + missing = True + if missing: + # Repair derived FTS rows only when support was missing, never on normal startup. + connection.exec_driver_sql("DELETE FROM pc_artifact_search_fts") + connection.exec_driver_sql( + "INSERT INTO pc_artifact_search_fts " + "(text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) " + "SELECT text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text " + "FROM pc_artifact_search_text" + ) diff --git a/src/powercontext/builtin/persistence/artifact_vector_space.py b/src/powercontext/builtin/persistence/artifact_vector_space.py new file mode 100644 index 0000000000..8d3dad7af8 --- /dev/null +++ b/src/powercontext/builtin/persistence/artifact_vector_space.py @@ -0,0 +1,155 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Server-owned OceanBase/seekDB VECTOR spaces, provisioned during database initialization.""" + +import re +from hashlib import sha256 + +from pyobvector import VECTOR +from sqlalchemy import ( + Column, + ForeignKeyConstraint, + Index, + MetaData, + Table, + UniqueConstraint, + inspect, + select, + text, + update, +) + +from powercontext.builtin.artifacts.memory.canonical import canonical_json +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTOR_SPACES_TABLE, + ARTIFACT_VECTORS_TABLE, + SHARED_METADATA, +) + + +def native_vector_table(space): + identity = space["space_id"] + if re.fullmatch(r"[0-9a-f]{32}", identity) is None or space["physical_table"] != f"pc_av_{identity}": + raise ValueError("invalid registered vector space table identity") # noqa: TRY003 + profile = EmbeddingProfile.model_validate(space["profile_json"]) + fingerprint = sha256(canonical_json(profile.model_dump(mode="json"))).hexdigest() + if ( + (profile.dimension, profile.distance, profile.normalization, fingerprint) + != (space["dimension"], space["distance"], space["normalization"], space["profile_fingerprint"]) + or profile.dimension < 1 + or space["storage_format_version"] != "float32-le-v1" + ): + raise ValueError("inconsistent vector space profile or storage format") # noqa: TRY003 + metadata = MetaData() + # Foreign key targets resolve in isolated metadata without provisioning them here. + for name in ("pc_artifacts", "pc_artifact_search_states"): + target = SHARED_METADATA.tables[name] + Table( + name, + metadata, + *(Column(column.name, column.type, primary_key=True) for column in target.primary_key.columns), + ) + return Table( + space["physical_table"], + metadata, + *( + Column( + column.name, + VECTOR(profile.dimension) if column.name == "embedding" else column.type, + primary_key=column.primary_key, + nullable=column.nullable, + ) + for column in ARTIFACT_VECTORS_TABLE.columns + ), + UniqueConstraint("build_id", "index_def_id", "artifact_id", "fragment_id", name=f"uq_av_{identity}"), + ForeignKeyConstraint( + ("scope_id", "family", "artifact_id", "revision"), + ("pc_artifacts.scope_id", "pc_artifacts.family", "pc_artifacts.artifact_id", "pc_artifacts.revision"), + ), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + Index(f"ix_av_{identity}", "scope_id", "family", "build_id", "space_id", "index_def_id", "artifact_id"), + ) + + +def verify_native_space(connection, space): # noqa: C901 - Verify one complete physical space contract. + table = native_vector_table(space) + inspector = inspect(connection) + if not inspector.has_table(table.name): + raise ValueError("vector space physical table is absent") # noqa: TRY003 + columns = {column["name"]: column for column in inspector.get_columns(table.name)} + if set(columns) != set(table.c.keys()): + raise ValueError("vector space physical columns differ from the registered format") # noqa: TRY003 + vector_type = connection.execute( + text( + "SELECT column_type FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name=:name AND column_name='embedding'" + ), + {"name": table.name}, + ).scalar_one() + if str(vector_type).lower().replace(" ", "") != f"vector({space['dimension']})": + raise ValueError("vector space physical dimension differs from its profile") # noqa: TRY003 + if inspector.get_pk_constraint(table.name)["constrained_columns"] != ["vector_id"]: + raise ValueError("vector space primary key is incompatible") # noqa: TRY003 + unique = {tuple(item["column_names"]) for item in inspector.get_unique_constraints(table.name)} + if ("build_id", "index_def_id", "artifact_id", "fragment_id") not in unique: + raise ValueError("vector space fragment identity constraint is absent") # noqa: TRY003 + foreign = { + (tuple(fk["constrained_columns"]), fk["referred_table"], tuple(fk["referred_columns"])) + for fk in inspector.get_foreign_keys(table.name) + } + for constraint in table.foreign_key_constraints: + if ( + tuple(constraint.columns.keys()), + constraint.referred_table.name, + tuple(element.column.name for element in constraint.elements), + ) not in foreign: + raise ValueError("vector space foreign key differs from the registered format") # noqa: TRY003 + actual_indexes = {tuple(item["column_names"]) for item in inspector.get_indexes(table.name)} + if ("scope_id", "family", "build_id", "space_id", "index_def_id", "artifact_id") not in actual_indexes: + raise ValueError("vector space search isolation index is absent") # noqa: TRY003 + collation_rows = connection.execute( + text( + "SELECT column_name, collation_name FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name=:name" + ), + {"name": table.name}, + ).all() + identities = { + "vector_id", + "scope_id", + "family", + "artifact_id", + "schema_version", + "index_def_id", + "build_id", + "fragment_id", + "input_hash", + "space_id", + } + if any(collation != "utf8mb4_bin" for name, collation in collation_rows if name in identities): + raise ValueError("vector space identities require binary collation") # noqa: TRY003 + if any(column["nullable"] for column in columns.values()): + raise ValueError("vector space columns must be nonnullable") # noqa: TRY003 + + +def provision_native_spaces(connection): + """Create registered spaces using the database initialization connection.""" + spaces = ( + connection + .execute(select(ARTIFACT_VECTOR_SPACES_TABLE).where(ARTIFACT_VECTOR_SPACES_TABLE.c.backend == "mysql")) + .mappings() + .all() + ) + for space in spaces: + table = native_vector_table(space) + table.create(connection, checkfirst=True) + verify_native_space(connection, space) + connection.execute( + update(ARTIFACT_VECTOR_SPACES_TABLE) + .where(ARTIFACT_VECTOR_SPACES_TABLE.c.space_id == space["space_id"]) + .values(provision_state="ready") + ) diff --git a/src/powercontext/builtin/persistence/custom_artifacts.py b/src/powercontext/builtin/persistence/custom_artifacts.py new file mode 100644 index 0000000000..e63e0e2c73 --- /dev/null +++ b/src/powercontext/builtin/persistence/custom_artifacts.py @@ -0,0 +1,87 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Durable declarations and JSON storage for user-defined Artifact families.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, JsonValue +from sqlalchemy import inspect +from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.schema import CreateColumn +from typing_extensions import override + +from powercontext.artifacts import Artifact +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.tables import ARTIFACT_PROCESSING_BINDING_STATES_TABLE + + +class StoredArtifactContent(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: str + content: dict[str, JsonValue] + + +def custom_artifact_repository( + family: str, sources: SourceRepository | None = None, *, prepared=None +) -> ArtifactRepository: + artifact_type = type("CustomArtifact", (Artifact[StoredArtifactContent],), {"family": family}) + return _CustomArtifactRepository((artifact_type,), sources=sources, prepared=prepared) + + +class _CustomArtifactRepository(ArtifactRepository): + def __init__(self, artifact_types, *, sources=None, prepared=None): + super().__init__(artifact_types, sources=sources) + self.prepared = prepared or {} + + @override + async def create(self, connection, scope_id, artifact_id, draft, /, **kwargs): + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher + + publisher = ArtifactProjectionPublisher(None) + prepared = self.prepared.get(artifact_id) + await publisher.validate_prepared( + connection, scope_id, draft.family, draft.content.schema_version, draft.content.content, prepared + ) + result = await super().create(connection, scope_id, artifact_id, draft, **kwargs) + await publisher.publish(connection, scope_id, result, prepared) + return result + + @override + async def lock_heads(self, connection, scope_id, refs, /): + from powercontext.builtin.persistence.artifact_indexes import lock_state + + for family in sorted({ref.family for ref in refs}): + await lock_state(connection, scope_id, family) + return await super().lock_heads(connection, scope_id, refs) + + @override + async def _revise(self, connection, scope_id, artifact, draft, **kwargs): + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher + + publisher = ArtifactProjectionPublisher(None) + prepared = self.prepared.get(artifact.artifact_id) + await publisher.validate_prepared( + connection, scope_id, draft.family, draft.content.schema_version, draft.content.content, prepared + ) + result = await super()._revise(connection, scope_id, artifact, draft, **kwargs) + await publisher.publish(connection, scope_id, result, prepared) + return result + + +async def ensure_artifact_plugin_schema(connection: AsyncConnection) -> None: + """Add registration fields to existing databases without replacing processing state.""" + + existing = await connection.run_sync( + lambda sync: {item["name"] for item in inspect(sync).get_columns(ARTIFACT_PROCESSING_BINDING_STATES_TABLE.name)} + ) + for column in ARTIFACT_PROCESSING_BINDING_STATES_TABLE.columns: + if column.name not in existing: + definition = str(CreateColumn(column).compile(dialect=connection.dialect)) + await connection.exec_driver_sql( + f"ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN {definition}" + ) diff --git a/src/powercontext/builtin/persistence/lexical_projection.py b/src/powercontext/builtin/persistence/lexical_projection.py index b1f564e3ed..c67aa3c902 100644 --- a/src/powercontext/builtin/persistence/lexical_projection.py +++ b/src/powercontext/builtin/persistence/lexical_projection.py @@ -19,7 +19,7 @@ from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncConnection -from powercontext.builtin.artifacts.search import analyze_text +from powercontext.builtin.artifacts.search import TEXT_ANALYZER_VERSION, analyze_text from powercontext.builtin.persistence.tables import ( LEXICAL_PROJECTION_VERSIONS_TABLE, TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, @@ -27,7 +27,7 @@ ) # Bump when document segmentation or token encoding changes; query stopwords do not affect projections. -LEXICAL_ANALYZER_VERSION = "jieba-0.42.1-words-v1" +LEXICAL_ANALYZER_VERSION = TEXT_ANALYZER_VERSION async def refresh_lexical_projections(connection: AsyncConnection, atomic_table: Table | None) -> None: diff --git a/src/powercontext/builtin/persistence/processing_intents.py b/src/powercontext/builtin/persistence/processing_intents.py index 0c451b94ad..8ef042192c 100644 --- a/src/powercontext/builtin/persistence/processing_intents.py +++ b/src/powercontext/builtin/persistence/processing_intents.py @@ -277,7 +277,13 @@ async def admit( state = ( ( await connection.execute( - select(state_table).where(state_table.c.binding_name == binding_name).with_for_update() + select( + state_table.c.scan_in_progress, + state_table.c.scan_generation, + state_table.c.scan_upper_pending_sequence, + ) + .where(state_table.c.binding_name == binding_name) + .with_for_update() ) ) .mappings() diff --git a/src/powercontext/builtin/persistence/records.py b/src/powercontext/builtin/persistence/records.py index 0fea04b3db..c5060c15d8 100644 --- a/src/powercontext/builtin/persistence/records.py +++ b/src/powercontext/builtin/persistence/records.py @@ -33,6 +33,7 @@ from powercontext.builtin.persistence.artifact_handles import ArtifactHandleRepository from powercontext.builtin.persistence.artifacts import ArtifactRepository, with_artifact_content from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import ( RepositoryNotFoundError, @@ -59,6 +60,7 @@ ) from powercontext.builtin.persistence.sources import SourceRepository, StoredSource from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE, SOURCE_JOURNAL_HEADS_TABLE, @@ -439,16 +441,22 @@ async def create_atomic_memories( async def get_artifact( self, scope_id: str, family: str, artifact_id: str, /, *, execution_context=None ) -> ArtifactRecord: - self._require_family(family) + repository = await self._read_repository(family) if family == "atomic-memory" and execution_context is not None: return await self._get_authorized_atomic_artifact(scope_id, artifact_id, None, execution_context) async with self._database.transaction() as connection: try: - artifact = await self._artifacts.latest(connection, scope_id, family, artifact_id) + artifact = await repository.latest(connection, scope_id, family, artifact_id) except RepositoryNotFoundError: raise BaseValueNotFoundError("artifact", (scope_id, family, artifact_id)) from None return _artifact_record(scope_id, artifact) + @property + def database(self) -> AsyncDatabase: + """The runtime-owned store shared by Server management services.""" + + return self._database + async def get_artifact_revision( self, scope_id: str, @@ -459,12 +467,12 @@ async def get_artifact_revision( *, execution_context=None, ) -> ArtifactRecord: - self._require_family(family) + repository = await self._read_repository(family) if family == "atomic-memory" and execution_context is not None: return await self._get_authorized_atomic_artifact(scope_id, artifact_id, revision, execution_context) async with self._database.transaction() as connection: try: - artifact = await self._artifacts.get( + artifact = await repository.get( connection, scope_id, ArtifactRef(family=family, artifact_id=artifact_id, revision=revision), @@ -502,13 +510,14 @@ async def list_artifact_revisions( rows, _, next_cursor = await self.revision_rows( connection, scope_id, family, artifact_id, limit=limit, cursor=cursor ) - artifacts = await self._artifacts.decode_many(connection, scope_id, rows) + repository = await self._read_repository(family, connection=connection) + artifacts = await repository.decode_many(connection, scope_id, rows) items = tuple(_artifact_collection_item(scope_id, artifact) for artifact in artifacts) return ArtifactRevisionPage(items=items, next_cursor=next_cursor) async def revision_rows(self, connection, scope_id, family, artifact_id, *, limit, cursor, columns=()): """Read a bounded immutable revision page through a borrowed transaction.""" - self._require_family(family) + await self._read_repository(family, connection=connection) _require_limit(limit) expected_cursor = { "version": 1, @@ -588,7 +597,7 @@ async def query_artifacts( cursor: str | None, tag_filter: TagFilter | None = None, ) -> ArtifactRecordPage: - self._require_family(family) + repository = await self._read_repository(family) _require_limit(limit) reader = self._topic_memory_list_reader if reader is not None and family == reader.family: @@ -650,7 +659,7 @@ async def query_artifacts( .all() ) selected_rows = rows[:limit] - artifacts = await self._artifacts.decode_many(connection, scope_id, selected_rows) + artifacts = await repository.decode_many(connection, scope_id, selected_rows) items = tuple(_artifact_collection_item(scope_id, artifact) for artifact in artifacts) next_cursor = None @@ -791,11 +800,22 @@ def _require_content_source(self, source_type: str) -> None: if source_type != CONTENT_SOURCE_NAME: raise InvalidBaseAccessRequestError("source_type", "must be content") - def _require_family(self, family: str) -> None: + async def _read_repository(self, family: str, *, connection: AsyncConnection | None = None) -> ArtifactRepository: if family == "memory": raise BaseOperationNotSupportedError("artifact_family", family, "legacy Memory collection read") - if family not in self._artifacts.families: + if family in self._artifacts.families: + return self._artifacts + if connection is None: + async with self._database.transaction() as borrowed: + return await self._read_repository(family, connection=borrowed) + known = await connection.scalar( + select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family) + .where(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family == family) + .limit(1) + ) + if known is None: raise InvalidBaseAccessRequestError("family", "must be a registered Artifact family") + return custom_artifact_repository(family) async def _get_source( self, @@ -869,9 +889,14 @@ def _artifact_record( artifact: Artifact[Any], ) -> ArtifactRecord: content = cast(dict[str, JsonValue], artifact.content.model_dump(mode="json", by_alias=True)) + schema_version = None + if isinstance(artifact.content, StoredArtifactContent): + schema_version = artifact.content.schema_version + content = artifact.content.content return ArtifactRecord( scope_id=scope_id, family=cast(Any, artifact.family), + schema_version=schema_version, artifact_id=artifact.artifact_id, revision=artifact.revision, content=content, @@ -895,9 +920,14 @@ def _artifact_created(scope_id: str, artifact: Artifact[Any]) -> ArtifactCreated def _artifact_collection_item(scope_id: str, artifact: Artifact[Any]) -> ArtifactCollectionItem: content = cast(dict[str, JsonValue], artifact.content.model_dump(mode="json", by_alias=True)) + schema_version = None + if isinstance(artifact.content, StoredArtifactContent): + schema_version = artifact.content.schema_version + content = artifact.content.content return ArtifactCollectionItem( scope_id=scope_id, family=cast(Any, artifact.family), + schema_version=schema_version, artifact_id=artifact.artifact_id, revision=artifact.revision, sources=artifact.lineage.sources, diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index d1e4170bc3..292877e832 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -25,8 +25,12 @@ async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: """Create missing tables and indexes using SQLAlchemy's native metadata support.""" + from powercontext.builtin.persistence.tables import ARTIFACT_VECTORS_TABLE + groups: dict[MetaData, list[Table]] = {} for table in tables: + if table is ARTIFACT_VECTORS_TABLE and connection.dialect.name != "sqlite": + continue groups.setdefault(table.metadata, []).append(table) for metadata, selected in groups.items(): @@ -37,3 +41,21 @@ async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) checkfirst=True, ) ) + + if any(table.name == "pc_artifact_search_text" for table in tables): + from powercontext.builtin.persistence.artifact_search_schema import ensure_search_support + + await connection.run_sync(ensure_search_support) + if connection.dialect.name != "sqlite" and any(table.name == "pc_artifact_vector_spaces" for table in tables): + from sqlalchemy import select + + from powercontext.builtin.persistence.artifact_indexes import SCHEMAS, register_definition, search_declaration + from powercontext.builtin.persistence.artifact_vector_space import provision_native_spaces + + rows = (await connection.execute(select(SCHEMAS))).mappings().all() + for row in rows: + declaration = search_declaration(row["declaration"]) + if declaration is not None: + for index in declaration.indexes: + await register_definition(connection, row["family"], row["schema_version"], index) + await connection.run_sync(provision_native_spaces) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index ba3cd56c56..e97516cc5b 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -39,7 +39,11 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository -from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE +from powercontext.builtin.persistence.tables import ( + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, +) from powercontext.builtin.sources.content import ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError @@ -171,6 +175,7 @@ async def add_with_status( intents = ArtifactProcessingIntentRepository() for binding_name in SOURCE_PROCESSING_BINDINGS: await intents.mark_dirty(connection, scope_id, binding_name) + await _mark_custom_bindings_dirty(connection, scope_id, intents) return StoredSource(ref=ref, value=source, journal_position=position), True async def get( @@ -523,3 +528,22 @@ def _require_identity(field: str, value: object, maximum: int) -> None: raise InvalidRepositoryArgumentError(field, "must be a non-empty trimmed string") if len(value) > maximum: raise InvalidRepositoryArgumentError(field, f"must not exceed {maximum} characters") + + +async def _mark_custom_bindings_dirty( + connection: AsyncConnection, + scope_id: str, + intents: ArtifactProcessingIntentRepository, +) -> None: + plugins = ( + await connection.execute( + select( + ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.binding_name, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.plugin_spec[("manifest", "scope_ids")].label("scope_ids"), + ).where(ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.plugin_spec.is_not(None)) + ) + ).all() + for plugin in plugins: + scope_ids = plugin.scope_ids + if scope_ids is None or scope_id in scope_ids: + await intents.mark_dirty(connection, scope_id, plugin.binding_name) diff --git a/src/powercontext/builtin/persistence/supervision.py b/src/powercontext/builtin/persistence/supervision.py index fcc3eced25..021d673084 100644 --- a/src/powercontext/builtin/persistence/supervision.py +++ b/src/powercontext/builtin/persistence/supervision.py @@ -291,9 +291,15 @@ async def load( .where(ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.binding_name == binding_name) .values(scan_generation=ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.scan_generation) ) - statement = select(ARTIFACT_PROCESSING_BINDING_STATES_TABLE).where( - ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.binding_name == binding_name - ) + table = ARTIFACT_PROCESSING_BINDING_STATES_TABLE + statement = select( + table.c.binding_name, + table.c.last_auto_wave_completed_at, + table.c.last_schedule_checkpoint_at, + table.c.scan_generation, + table.c.scan_in_progress, + table.c.scan_upper_pending_sequence, + ).where(table.c.binding_name == binding_name) if for_update: statement = statement.with_for_update() row = (await connection.execute(statement)).mappings().one_or_none() diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 2793f9d9d0..4bfc460f0e 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -15,6 +15,7 @@ """Shared dialect-neutral relational tables.""" from sqlalchemy import ( + JSON, BigInteger, Boolean, CheckConstraint, @@ -34,6 +35,7 @@ ) from sqlalchemy.dialects.mysql import BINARY, MEDIUMBLOB, MEDIUMTEXT, VARCHAR +from powercontext.builtin.persistence.artifact_search_schema import artifact_search_tables from powercontext.limits import ( MAX_ARTIFACT_FAMILY_LENGTH, MAX_ARTIFACT_HANDLE_KEY_LENGTH, @@ -547,9 +549,24 @@ def _entry_text_type(): Column("scan_generation", BigInteger, nullable=False, server_default="0"), Column("scan_in_progress", Boolean, nullable=False, server_default="0"), Column("scan_upper_pending_sequence", BigInteger), + Column("plugin_id", String(128)), + Column("plugin_family", String(128)), + Column("plugin_spec", JSON(none_as_null=True)), + Column("binding_generation", BigInteger, nullable=False, server_default="0"), + Column("desired_state", String(16)), + Column("phase", String(16)), + Column("last_error", String(1024)), CheckConstraint("scan_generation >= 0", name="ck_pc_processing_scan_generation"), ) +ARTIFACT_FAMILY_SCHEMAS_TABLE = Table( + "pc_artifact_family_schemas", + SHARED_METADATA, + Column("family", identity_string(128), primary_key=True), + Column("schema_version", identity_string(128), primary_key=True), + Column("declaration", JSON, nullable=False), +) + ARTIFACT_PROCESSING_SEQUENCES_TABLE = Table( "pc_artifact_processing_sequences", SHARED_METADATA, @@ -1037,6 +1054,7 @@ def _entry_text_type(): SHARED_TABLES = ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE, ARTIFACTS_TABLE, @@ -1400,3 +1418,17 @@ def _entry_text_type(): LEXICAL_PROJECTION_VERSIONS_TABLE, ) ) + + +# Search builds are independent of generation bindings. + +( + ARTIFACT_VECTOR_SPACES_TABLE, + ARTIFACT_VECTOR_INDEXES_TABLE, + ARTIFACT_SEARCH_STATES_TABLE, + ARTIFACT_INDEX_COVERAGE_TABLE, + ARTIFACT_INDEX_CHANGES_TABLE, + ARTIFACT_VECTORS_TABLE, + ARTIFACT_SEARCH_TEXT_TABLE, +) = ARTIFACT_SEARCH_TABLES = artifact_search_tables(SHARED_METADATA, identity_string) +BUILTIN_TABLES += ARTIFACT_SEARCH_TABLES diff --git a/src/powercontext/builtin/records.py b/src/powercontext/builtin/records.py index 7fdc985c89..0b9bdca9b9 100644 --- a/src/powercontext/builtin/records.py +++ b/src/powercontext/builtin/records.py @@ -30,7 +30,7 @@ from powercontext.builtin.tags import ArtifactTagSet, TagFilter, TagQuery, TagQueryPage, TagTarget BaseArtifactFamily = Literal["atomic-memory", "experience", "skill", "handoff", "profile", "prompt", "topic-memory"] -ArtifactReadFamily = Literal["atomic-memory", "experience", "skill", "handoff", "profile", "prompt", "topic-memory"] +ArtifactReadFamily = str class _RecordModel(BaseModel): @@ -88,6 +88,7 @@ class ArtifactRecord(_RecordModel): scope_id: str family: ArtifactReadFamily + schema_version: str | None = None artifact_id: str revision: int content: dict[str, JsonValue] @@ -105,6 +106,7 @@ class ArtifactCollectionItem(_RecordModel): scope_id: str family: ArtifactReadFamily + schema_version: str | None = None artifact_id: str revision: int sources: tuple[SourceRef, ...] diff --git a/src/powercontext/builtin/runtime/artifact_plugins.py b/src/powercontext/builtin/runtime/artifact_plugins.py new file mode 100644 index 0000000000..bfc057c23a --- /dev/null +++ b/src/powercontext/builtin/runtime/artifact_plugins.py @@ -0,0 +1,346 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Execute validated Artifact plugins in an owned generation Worker.""" + +from __future__ import annotations + +import asyncio +import json +import logging +from contextlib import AsyncExitStack +from dataclasses import dataclass +from typing import Any + +from jsonschema import Draft202012Validator +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext._logging import log_safely +from powercontext.artifact_plugins import ( + ArtifactDraft, + GenerationBatch, + GenerationContext, + GenerationSource, +) +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.artifact_plugin_validation import ( + _MAX_INPUT_BYTES, + _MAX_OUTPUT_BYTES, + _PluginManifest, + code_fingerprint, + describe_module, + load_plugin, +) +from powercontext.builtin.persistence.artifacts import RepositoryArtifactDraft +from powercontext.builtin.persistence.cursors import SourceCursorRepository +from powercontext.builtin.persistence.custom_artifacts import ( + StoredArtifactContent, + custom_artifact_repository, +) +from powercontext.builtin.persistence.generation_sources import GenerationSourceAccess +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, +) +from powercontext.builtin.runtime.artifact_reads import ArtifactReadSession +from powercontext.builtin.runtime.config import BuiltinConfig, DatabaseConfig, RuntimeConfig +from powercontext.builtin.runtime.processing_contracts import ( + ArtifactProcessingWorkAssignment, + ArtifactProcessingWorkerCompletion, +) +from powercontext.builtin.runtime.processing_execution import InvocationAlreadyHandled, ScopeInvocation +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, SourceCursor +from powercontext.sources import SourceProjectionKey + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ArtifactPluginWorkerSpec: + database: DatabaseConfig + binding_name: str + generation: int + definition: dict[str, Any] + security: dict[str, Any] | None = None + inference_config: Any = None + runtime_config: Any = None + + +async def require_current_binding(connection: AsyncConnection, spec: ArtifactPluginWorkerSpec, scope_id: str) -> None: + table = ARTIFACT_PROCESSING_BINDING_STATES_TABLE + locked = await connection.execute( + update(table) + .where( + table.c.binding_name == spec.binding_name, + table.c.binding_generation == spec.generation, + table.c.phase == "enabled", + ) + .values(binding_generation=table.c.binding_generation) + ) + scope_ids = spec.definition["manifest"]["scope_ids"] + if locked.rowcount != 1 or (scope_ids is not None and scope_id not in scope_ids): + raise ValueError("plugin invocation has been revoked") # noqa: TRY003 + + +def run_artifact_plugin_worker( + spec: ArtifactPluginWorkerSpec, + assignment: ArtifactProcessingWorkAssignment, +) -> ArtifactProcessingWorkerCompletion: + return asyncio.run(_process_window(spec, assignment)) + + +async def _restore_access_families(database) -> None: + from powercontext.server.authz.profiles import register_custom_artifact_access_family + + async with database.transaction() as connection: + families = (await connection.execute(select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family).distinct())).scalars().all() + for family in families: + register_custom_artifact_access_family(family) + + +async def _process_window(spec: ArtifactPluginWorkerSpec, assignment: ArtifactProcessingWorkAssignment): # noqa: C901 - One authorized window with preparation outside its commit. + # Server authorization is reconstructed in this child using deployment identity. + from powercontext.server.authz import AccessAction, ResourceRef + from powercontext.server.processing_security import open_worker_security + + definition = spec.definition + declaration = definition["declaration"] + if assignment.binding_name != spec.binding_name or assignment.artifact_family != declaration["family"]: + raise ValueError("assignment does not match the registered plugin") # noqa: TRY003 + sources = SourceRepository(BUILTIN_SOURCE_REGISTRY) + cursors = SourceCursorRepository() + async with AsyncExitStack() as resources: + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher + from powercontext.builtin.runtime.composition import ( + _embedding_models, + compose_artifact_search, + open_builtin_contexts, + ) + + embedding = None + if spec.inference_config is not None: + embedding, _ = await _embedding_models(spec.inference_config, resources, None) + config = BuiltinConfig(database=spec.database, runtime=spec.runtime_config or RuntimeConfig()) + storage = await resources.enter_async_context( + open_builtin_contexts(config, embedding_model=embedding, _topic_memory_worker=True) + ) + search_service, _ = compose_artifact_search(storage, embedding) + publisher = ArtifactProjectionPublisher(storage.database, embedding) + await _restore_access_families(storage.database) + security = await resources.enter_async_context(open_worker_security(spec.security, storage.database)) + + async def authorize(connection): + await require_current_binding(connection, spec, assignment.scope_id) + if security is not None: + await security.authorize_transaction(connection, assignment.scope_id) + await security.access.with_connection(connection).require( + security.principal, + AccessAction.SCOPE_READ, + ResourceRef.scope(assignment.scope_id), + context=security.context, + ) + + invocation = ScopeInvocation(assignment, authorize_transaction=authorize) + try: + async with storage.database.transaction() as connection: + await invocation.start(connection) + cursor = await cursors.load(connection, assignment.scope_id, spec.binding_name) + after = 0 if cursor is None else cursor.cursor.sequence + high = await sources.journal_position(connection, assignment.scope_id) + through = min(high, after + definition["manifest"]["max_sources"]) + rows = await GenerationSourceAccess(sources).list_window_for_generation( + connection, + assignment.scope_id, + after=after, + through=through, + ) + # Desired index builds belong to maintenance. Generation prepares only the + # current serving projections, so an unavailable future index cannot block writes. + projections = { + name: SourceProjectionKey.model_validate(key) for name, key in declaration["projections"].items() + } + batch = GenerationBatch( + sources=tuple( + GenerationSource( + ref=row.ref, + projection=projections[row.ref.source_type], + value=BUILTIN_SOURCE_REGISTRY.project(row.value, projections[row.ref.source_type]), + ) + for row in rows + if row.ref.source_type in projections + ) + ) + if ( + len( + json.dumps( + [source.model_dump(mode="json") for source in batch.sources], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ) + > _MAX_INPUT_BYTES + ): + raise ValueError("input evidence exceeds the window budget") # noqa: TRY003 + source_evidence = frozenset((source.ref.source_type, source.ref.source_id) for source in batch.sources) + artifact_evidence = ArtifactReadSession( + storage.database, + assignment.scope_id, + {declaration["family"], *declaration.get("artifact_families", ())}, + access=None if security is None else security.access, + principal=None if security is None else security.principal, + audit=None if security is None else security.context, + guard=invocation.guard, + search_service=search_service, + ) + + drafts = [] + if batch.sources: + if code_fingerprint(definition["directory"]) != declaration["fingerprint"]: + raise ValueError("plugin files changed after registration") # noqa: TRY003 + manifest = _PluginManifest.model_validate(definition["manifest"]) + module = load_plugin(definition["directory"], manifest) + if describe_module(module, manifest, definition["directory"]) != { + "manifest": definition["manifest"], + "declaration": declaration, + }: + raise ValueError("Worker declaration differs from the registered definition") # noqa: TRY003 + ctx = GenerationContext( + assignment.scope_id, + resources, + artifact_evidence.read, + list_artifacts=artifact_evidence.list, + search_artifacts=artifact_evidence.search, + ) + pipeline = await module.create_pipeline( + ctx, module.Config.model_validate_json(json.dumps(manifest.config, ensure_ascii=False)) + ) + drafts = await pipeline.generate(batch) + validated = _validate_drafts(declaration, source_evidence, artifact_evidence.refs, drafts) + prepared = {} + for draft in validated: + prepared[draft.artifact_id] = await publisher.prepare( + assignment.scope_id, draft.family, draft.schema_version, draft.content + ) + await _commit_window( + storage.database, + spec, + assignment, + declaration, + invocation, + sources, + cursor, + through, + source_evidence, + artifact_evidence.refs, + drafts, + security, + prepared, + ) + except InvocationAlreadyHandled: + pass + return ArtifactProcessingWorkerCompletion() + + +def _validate_drafts(declaration, source_evidence, artifact_evidence, drafts) -> list[ArtifactDraft]: + if not isinstance(drafts, list): + raise TypeError("plugin must return a list of drafts") # noqa: TRY003 + validated = [ArtifactDraft.model_validate(draft) for draft in drafts] + if ( + len(json.dumps([draft.model_dump(mode="json") for draft in validated], ensure_ascii=False).encode()) + > _MAX_OUTPUT_BYTES + ): + raise ValueError("output exceeds the window budget") # noqa: TRY003 + validator = Draft202012Validator(declaration["content_schema"]) + identities = set() + for draft in validated: + if (draft.family, draft.schema_version) != (declaration["family"], declaration["schema_version"]): + raise ValueError("draft does not match the registered family/schema") # noqa: TRY003 + if draft.artifact_id in identities or not { + (ref.source_type, ref.source_id) for ref in draft.source_refs + }.issubset(source_evidence): + raise ValueError("duplicate artifact or Source evidence outside this invocation") # noqa: TRY003 + if not {(ref.family, ref.artifact_id, ref.revision) for ref in draft.artifact_refs}.issubset(artifact_evidence): + raise ValueError("Artifact evidence was not read through this invocation") # noqa: TRY003 + if ( + draft.expected_revision > 0 + and (draft.family, draft.artifact_id, draft.expected_revision) not in artifact_evidence + ): + raise ValueError("update revision must have been read through this invocation") # noqa: TRY003 + identities.add(draft.artifact_id) + validator.validate(draft.content) + return validated + + +async def _commit_window( + database, + spec, + assignment, + declaration, + invocation, + sources, + cursor, + through, + source_evidence, + artifact_evidence, + drafts, + security, + prepared_indexes=None, +): + from powercontext.server.authz import AccessAction, ResourceRef + + validated = _validate_drafts(declaration, source_evidence, artifact_evidence, drafts) + repository = custom_artifact_repository(declaration["family"], sources, prepared=prepared_indexes) + async with database.transaction() as connection: + await invocation.guard(connection) + for draft in validated: + resource = ResourceRef.artifact(assignment.scope_id, family=draft.family, artifact_id=draft.artifact_id) + bound = None if security is None else security.access.with_connection(connection) + if bound is not None: + for ref in draft.artifact_refs: + await bound.require( + security.principal, + AccessAction.ARTIFACT_READ, + ResourceRef.artifact(assignment.scope_id, family=ref.family, artifact_id=ref.artifact_id), + context=security.context, + ) + prepared = RepositoryArtifactDraft( + family=draft.family, + content=StoredArtifactContent(schema_version=draft.schema_version, content=draft.content), + sources=draft.source_refs, + artifacts=draft.artifact_refs, + ) + if draft.expected_revision == 0: + await repository.create(connection, assignment.scope_id, draft.artifact_id, prepared) + if bound is not None: + await bound.establish_artifact_owner( + resource, + security.principal, + idempotency_key=f"background-custom-owner:{assignment.scope_id}:{draft.family}:{draft.artifact_id}", + context=security.context, + ) + else: + if bound is not None: + await bound.require( + security.principal, AccessAction.ARTIFACT_WRITE, resource, context=security.context + ) + previous = await repository.get( + connection, + assignment.scope_id, + ArtifactRef(family=draft.family, artifact_id=draft.artifact_id, revision=draft.expected_revision), + ) + await repository.revise(connection, assignment.scope_id, previous, prepared) + if through > (0 if cursor is None else cursor.cursor.sequence): + await SourceCursorRepository().save( + connection, + assignment.scope_id, + spec.binding_name, + SourceCursor(sequence=through), + expected_generation=None if cursor is None else cursor.generation, + ) + high = await sources.journal_position(connection, assignment.scope_id) + await invocation.complete(connection, remaining_work=high > through) + log_safely(logger, logging.DEBUG, "Artifact plugin window committed") diff --git a/src/powercontext/builtin/runtime/artifact_processing.py b/src/powercontext/builtin/runtime/artifact_processing.py index a6c42932c8..d93073ff51 100644 --- a/src/powercontext/builtin/runtime/artifact_processing.py +++ b/src/powercontext/builtin/runtime/artifact_processing.py @@ -131,6 +131,8 @@ class ArtifactProcessingBinding: # Applies to every invocation, including explicit requests and retries. # Domains inspect durable metadata here; they retain their Worker guards. work_block: Callable[[AsyncConnection, str, str], Awaitable[ArtifactProcessingBlock | None]] | None = None + # Hot-managed definitions can revoke a Scope even for an older accepted request. + invocation_scope_filter: Callable[[AsyncConnection, tuple[str, ...]], Awaitable[frozenset[str]]] | None = None # Receives one fixed telemetry summary after every worker execution attempt. # It must not affect domain scheduling or completion semantics. worker_model_usage_observer: Callable[[ArtifactProcessingModelUsageDelta | None], None] | None = None @@ -551,9 +553,14 @@ def __init__( self._rotation = 0 self._lease_lost = False self._unreaped: list[ArtifactProcessingWorkerHandle] = [] + self._binding_changes: asyncio.Queue[tuple[str, ArtifactProcessingBinding | None, asyncio.Future[None]]] = ( + asyncio.Queue() + ) @property def status(self) -> ArtifactProcessingSupervisorStatus: + if not self._families: + return ArtifactProcessingSupervisorStatus.DISABLED if self._fence is not None and any(state.degraded for state in self._families.values()): return ArtifactProcessingSupervisorStatus.DEGRADED return self._status @@ -645,6 +652,42 @@ def wake(self, binding_name: str | None = None) -> None: state.discovery_pending = True self._wake.set() + async def replace_binding(self, name: str, binding: ArtifactProcessingBinding | None) -> None: + """Change one global binding at a scheduling boundary and reap its old children.""" + + if self._supervisor_group != "global" or self._task is None or self._stop.is_set(): + raise ValueError("hot registration requires a running global Supervisor") # noqa: TRY003 + if binding is not None and binding.binding_name != name: + raise ValueError("replacement binding identity must match") # noqa: TRY003 + done = asyncio.get_running_loop().create_future() + self._binding_changes.put_nowait((name, binding, done)) + self._wake.set() + await asyncio.shield(done) + + async def _apply_binding_changes(self) -> None: + while not self._binding_changes.empty(): + name, binding, done = self._binding_changes.get_nowait() + try: + state = self._families.get(name) + if state is not None: + tasks = [worker.task for worker in state.running.values()] + for task in tasks: + task.cancel() + results = await asyncio.gather(*tasks, return_exceptions=True) + for result in results: + if isinstance(result, _WorkerTerminationError): + raise result # noqa: TRY301 - propagate failed child reaping to leadership revocation + del self._families[name] + if binding is not None: + if any(item.binding.artifact_family == binding.artifact_family for item in self._families.values()): + raise ValueError("an Artifact family already has a binding") # noqa: TRY003, TRY301 + self._families[name] = _FamilyState(binding) + done.set_result(None) + except BaseException as error: + done.set_exception(error) + if isinstance(error, (asyncio.CancelledError, _WorkerTerminationError)): + raise + async def close(self) -> None: self._stop.set() self._wake.set() @@ -660,6 +703,7 @@ async def _run(self) -> None: # noqa: C901 - lifecycle and cancellation boundar while not self._stop.is_set(): self._wake.clear() try: + await self._apply_binding_changes() if self._lease_lost: await self._lose_leadership() if self._fence is None: @@ -688,6 +732,9 @@ async def _run(self) -> None: # noqa: C901 - lifecycle and cancellation boundar except TimeoutError: pass finally: + while not self._binding_changes.empty(): + _, _, done = self._binding_changes.get_nowait() + done.set_exception(RuntimeError("Supervisor stopped before applying the binding")) cleanup = asyncio.create_task( self._lose_leadership(), name=f"artifact-supervisor-cleanup-{self._supervisor_group}" ) @@ -1040,7 +1087,7 @@ def _defer( state.pending_since.setdefault(scope, time.time()) state.retries[scope] = _RetryState(failures, deadline, block, request_generation, probe_error) - async def _dispatch(self, state: _FamilyState) -> None: + async def _dispatch(self, state: _FamilyState) -> None: # noqa: C901 - ordered admission and worker scheduling # Rejections do not occupy slots. Bound their admission work so a page # of slow metadata probes cannot monopolize the shared control loop. loop = asyncio.get_running_loop() @@ -1055,6 +1102,10 @@ async def _dispatch(self, state: _FamilyState) -> None: async with self._database.transaction() as connection: fence = self._require_current_fence() await self._leases.require_fence(connection, fence) + allowed = state.binding.invocation_scope_filter + if allowed is not None and scope not in await allowed(connection, (scope,)): + state.retries.pop(scope, None) + continue row = await self._intents.load(connection, scope, state.binding.binding_name, for_update=True) if row is None or row.requested_generation <= row.handled_generation: state.retries.pop(scope, None) @@ -1637,6 +1688,8 @@ def __init__(self, supervisors: Sequence[ArtifactProcessingSupervisor]) -> None: @property def status(self) -> ArtifactProcessingSupervisorStatus: states = [item.status for item in self.supervisors] + if all(state is ArtifactProcessingSupervisorStatus.DISABLED for state in states): + return ArtifactProcessingSupervisorStatus.DISABLED if ArtifactProcessingSupervisorStatus.DEGRADED in states: return ArtifactProcessingSupervisorStatus.DEGRADED if ArtifactProcessingSupervisorStatus.LEADER in states: @@ -1657,6 +1710,11 @@ def wake(self, binding_name: str | None = None) -> None: for supervisor in self.supervisors: supervisor.wake(binding_name) + async def replace_binding(self, name: str, binding: ArtifactProcessingBinding | None) -> None: + if len(self.supervisors) != 1: + raise ValueError("hot registration requires one global Supervisor") # noqa: TRY003 + await self.supervisors[0].replace_binding(name, binding) + async def close(self) -> None: await asyncio.gather(*(item.close() for item in self.supervisors)) diff --git a/src/powercontext/builtin/runtime/artifact_reads.py b/src/powercontext/builtin/runtime/artifact_reads.py new file mode 100644 index 0000000000..cf67b8ec3d --- /dev/null +++ b/src/powercontext/builtin/runtime/artifact_reads.py @@ -0,0 +1,194 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Invocation-scoped Artifact reads and shared custom-family FTS adapters.""" + +from contextlib import AsyncExitStack, asynccontextmanager +from typing import get_args + +from pydantic import JsonValue, RootModel +from sqlalchemy import select + +from powercontext.artifact_plugins import ArtifactReferencePage, ArtifactSnapshot +from powercontext.artifacts import Artifact, ArtifactRef +from powercontext.artifacts.search import ( + ArtifactSearchExecutionContext, +) +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE, ARTIFACT_HEADS_TABLE +from powercontext.builtin.records import BaseArtifactFamily, BaseValueNotFoundError, InvalidBaseAccessRequestError +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService +from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher +from powercontext.server.authz import AccessAction, ResourceRef + + +class _StoredJsonContent(RootModel[dict[str, JsonValue]]): + pass + + +class ArtifactReadSession: + """Constrain reads to one Scope and record exact bodies eligible for generation lineage.""" + + def __init__( + self, + database, + scope_id, + families, + *, + access=None, + principal=None, + audit=None, + guard=None, + embedding_model=None, + search_service=None, + ): + self.database, self.scope_id = database, scope_id + self.families = frozenset(families) + self.access, self.principal, self.audit, self.guard = access, principal, audit, guard + self.refs: set[tuple[str, str, int]] = set() + self.cursor = SignedCursorCodec() + self.search_service = search_service or ArtifactSearchService( + known_families=get_args(BaseArtifactFamily), + fallback=lambda family: CustomArtifactSearcher(database, family, embedding_model), + ) + + def _family(self, family): + if family not in self.families: + raise InvalidBaseAccessRequestError("family", "is outside the invocation's declared read families") + + @asynccontextmanager + async def _connection(self, family): + self._family(family) + async with AsyncExitStack() as resources: + if self.access is not None: + await resources.enter_async_context(self.access.defer_decision_audit()) + connection = await resources.enter_async_context(self.database.transaction()) + await connection.exec_driver_sql("SAVEPOINT pc_artifact_read") + if self.guard is not None: + await self.guard(connection) + if self.access is not None: + await self.access.for_read(connection).require( + self.principal, + AccessAction.SCOPE_READ, + ResourceRef.scope(self.scope_id), + context=self.audit, + ) + yield connection + + async def _repository(self, connection, family): + known = await connection.scalar( + select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family) + .where( + ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family == family, + ) + .limit(1) + ) + if known is not None: + return custom_artifact_repository(family) + if family in get_args(BaseArtifactFamily): + artifact_type = type("StoredArtifact", (Artifact[_StoredJsonContent],), {"family": family}) + return ArtifactRepository((artifact_type,)) + raise BaseValueNotFoundError("artifact_family", family) + + async def read(self, family, artifact_id, revision): + async with self._connection(family) as connection: + repository = await self._repository(connection, family) + try: + artifact = ( + await repository.latest(connection, self.scope_id, family, artifact_id) + if revision is None + else await repository.get( + connection, + self.scope_id, + ArtifactRef(family=family, artifact_id=artifact_id, revision=revision), + ) + ) + except RepositoryNotFoundError: + if revision is None: + return None + raise BaseValueNotFoundError("artifact_revision", (family, artifact_id, revision)) from None + if self.access is not None: + await self.access.for_read(connection).require( + self.principal, + AccessAction.ARTIFACT_READ, + ResourceRef.artifact(self.scope_id, family=family, artifact_id=artifact_id), + context=self.audit, + ) + ref = artifact.as_ref() + self.refs.add((ref.family, ref.artifact_id, ref.revision)) + custom = isinstance(artifact.content, StoredArtifactContent) + return ArtifactSnapshot( + ref=ref, + lineage=artifact.lineage, + schema_version=artifact.content.schema_version if custom else None, + content=artifact.content.content if custom else artifact.content.model_dump(mode="json", by_alias=True), + source_refs=artifact.lineage.sources, + artifact_refs=artifact.lineage.artifacts, + ) + + async def list(self, family, *, limit=50, cursor=None): + if type(limit) is not int or not 1 <= limit <= 100: + raise InvalidBaseAccessRequestError("limit", "must be between 1 and 100") + expected = {"scope": self.scope_id, "family": family, "limit": limit} + after = self.cursor.after_text(cursor, expected) + selected = [] + async with self._connection(family) as connection: + await self._repository(connection, family) + bound = None if self.access is None else self.access.for_read(connection) + while len(selected) <= limit: + rows = ( + await connection.execute( + select(ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision) + .where( + ARTIFACT_HEADS_TABLE.c.scope_id == self.scope_id, + ARTIFACT_HEADS_TABLE.c.family == family, + ARTIFACT_HEADS_TABLE.c.lifecycle_state == "active", + ARTIFACT_HEADS_TABLE.c.artifact_id > after, + ) + .order_by(ARTIFACT_HEADS_TABLE.c.artifact_id) + .limit(100) + ) + ).all() + if not rows: + break + for artifact_id, revision in rows: + after = artifact_id + if bound is not None: + decision = await bound.check( + self.principal, + AccessAction.ARTIFACT_READ, + ResourceRef.artifact(self.scope_id, family=family, artifact_id=artifact_id), + context=self.audit, + ) + if not decision.allowed: + continue + selected.append(ArtifactRef(family=family, artifact_id=artifact_id, revision=revision)) + if len(selected) > limit: + break + items = tuple(selected[:limit]) + return ArtifactReferencePage( + items=items, + next_cursor=self.cursor.encode(expected, items[-1].artifact_id) if len(selected) > limit else None, + ) + + async def search(self, family, query=None, **parameters): + self._family(family) + async with self._connection(family): + pass + payload = dict(parameters) + if query is not None: + payload["query"] = query + outcome = await self.search_service.search( + self.scope_id, + family, + payload, + execution_context=ArtifactSearchExecutionContext( + access=self.access, principal=self.principal, audit=self.audit, trusted_local=self.access is None + ), + ) + return outcome diff --git a/src/powercontext/builtin/runtime/artifact_search.py b/src/powercontext/builtin/runtime/artifact_search.py index 226a420e10..ddc5554b3a 100644 --- a/src/powercontext/builtin/runtime/artifact_search.py +++ b/src/powercontext/builtin/runtime/artifact_search.py @@ -66,8 +66,9 @@ async def invoke( class ArtifactSearchService: """Dispatch repository-known Families without knowing their concrete models.""" - def __init__(self, *, known_families: Iterable[str]) -> None: + def __init__(self, *, known_families: Iterable[str], fallback=None) -> None: self._known_families = frozenset(known_families) + self._fallback = fallback self._bindings: dict[str, _SearcherBinding[Any, ArtifactSearchOutcome]] = {} def register( @@ -105,6 +106,8 @@ async def search( def _binding(self, family: str) -> _SearcherBinding[Any, ArtifactSearchOutcome]: if family not in self._known_families: + if self._fallback is not None: + return _SearcherBinding(self._fallback(family), None) raise ArtifactSearchFamilyNotFound(family) binding = self._bindings.get(family) if binding is None: diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index f02631453f..87da7e2be7 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -104,6 +104,7 @@ ensure_supported_atomic_memory_schema, ) from powercontext.builtin.persistence.bootstrap_receipts import BootstrapReceiptRepository +from powercontext.builtin.persistence.custom_artifacts import ensure_artifact_plugin_schema from powercontext.builtin.persistence.experience_index import ensure_artifact_head_searchable_text from powercontext.builtin.persistence.lexical_projection import refresh_lexical_projections from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex @@ -745,21 +746,8 @@ async def run_profile(scope_id, high): if spec.processing_family in registered_families or (spec.family is None and registered_families) ) topic_memory_processing_available = _topic_memory_processing_available(config, processing_bindings) - artifact_search = ArtifactSearchService(known_families=contexts.repositories.artifacts.families) - artifact_search.register( - ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, contexts.experience_index) - ) - artifact_search.register(contexts.list_skills) - topic_memory_searcher = TopicMemorySearcher( - search=contexts.search_topic_memories, - embedding_model=configured_embedding, - observer=topic_memory_search_observer, - capabilities=contexts.topic_memory_index.capabilities, - ) - artifact_search.register(topic_memory_searcher, embedding_purpose=ModelUsagePurpose.TOPIC_MEMORY_RECALL) - artifact_search.register( - AtomicArtifactSearcher(application=contexts.atomic_memory), - embedding_purpose=ModelUsagePurpose.MEMORY_RECALL, + artifact_search, topic_memory_searcher = compose_artifact_search( + contexts, configured_embedding, observer=topic_memory_search_observer ) runtime = await resources.enter_async_context( BuiltinRuntime( @@ -1088,6 +1076,32 @@ async def _open_artifact_processing_supervisor( yield ArtifactProcessingSupervisors(tuple(supervisors)) +def compose_artifact_search(contexts, embedding_model=None, *, observer=None): + """Use the same Family-owned Searchers in Server, SDK and generation Workers.""" + from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher + + artifact_search = ArtifactSearchService( + known_families=contexts.repositories.artifacts.families, + fallback=lambda family: CustomArtifactSearcher(contexts.database, family, embedding_model), + ) + artifact_search.register( + ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, contexts.experience_index) + ) + artifact_search.register(contexts.list_skills) + topic_memory_searcher = TopicMemorySearcher( + search=contexts.search_topic_memories, + embedding_model=embedding_model, + observer=observer, + capabilities=contexts.topic_memory_index.capabilities, + ) + artifact_search.register(topic_memory_searcher, embedding_purpose=ModelUsagePurpose.TOPIC_MEMORY_RECALL) + artifact_search.register( + AtomicArtifactSearcher(application=contexts.atomic_memory), + embedding_purpose=ModelUsagePurpose.MEMORY_RECALL, + ) + return artifact_search, topic_memory_searcher + + @asynccontextmanager async def open_builtin_contexts( config: BuiltinConfig, @@ -1130,13 +1144,14 @@ async def open_builtin_contexts( async with SQLiteProfile.open( database, tables=BUILTIN_TABLES + topic_index.tables + atomic_index.tables, - load_vector_extension=embedding_model is not None, + load_vector_extension=True, ) as profile: async with profile.database.transaction() as connection: # This block is SQLite-only. Reserve SQLite's single-writer slot # before startup schema reads, so concurrent Runtime openings cannot # both establish a read snapshot and fail while upgrading to a write. await connection.exec_driver_sql("BEGIN IMMEDIATE") + await ensure_artifact_plugin_schema(connection) await bootstrap_processing_schema(connection, canonical_processing_manifest(config)) await assert_processing_schema_ready(connection, canonical_processing_manifest(config)) await ensure_skill_distribution_schema(connection) @@ -1223,6 +1238,7 @@ async def open_builtin_contexts( async with profile_context as profile: async with profile.database.transaction() as connection: await ensure_supported_atomic_memory_schema(connection) + await ensure_artifact_plugin_schema(connection) await bootstrap_processing_schema(connection, canonical_processing_manifest(config)) await assert_processing_schema_ready(connection, canonical_processing_manifest(config)) await ensure_skill_distribution_schema(connection) diff --git a/src/powercontext/builtin/runtime/custom_artifact_search.py b/src/powercontext/builtin/runtime/custom_artifact_search.py new file mode 100644 index 0000000000..1d840393fa --- /dev/null +++ b/src/powercontext/builtin/runtime/custom_artifact_search.py @@ -0,0 +1,445 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Declaration-driven text/vector/hybrid search returning complete exact Artifacts.""" + +from __future__ import annotations + +import math +import struct +from contextlib import AsyncExitStack +from dataclasses import dataclass +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, RootModel, field_validator +from sqlalchemy import bindparam, select, text +from sqlalchemy.exc import SQLAlchemyError + +from powercontext.artifacts import Artifact, ArtifactRef +from powercontext.artifacts.fusion import ( + FusionCandidate, + FusionChannel, + FusionSelection, + RrfParameters, + fuse_rrf, + parse_fusion_parameters, +) +from powercontext.artifacts.indexes import VectorIndex +from powercontext.artifacts.search import ( + ArtifactSearchExecutionContext, + ArtifactSearchFamilyNotFound, + ArtifactSearchMatch, + ArtifactSearchQuery, + ArtifactSearchUnavailable, + ArtifactSearchUnsupported, + ChannelScore, +) +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.artifacts.search import analyze_fts_query, fts_match_query, lexical_search_score +from powercontext.builtin.inference.protocols import embed_query +from powercontext.builtin.persistence.artifact_indexes import ( + DEFINITIONS, + SCHEMAS, + STATES, + entries, + key, + parse_index, + projection_table, + search_declaration, + vector_bytes, +) +from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository +from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACT_PROCESSING_BINDING_STATES_TABLE +from powercontext.server.authz import AccessAction, ResourceRef + + +class ArtifactQueryVector(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, allow_inf_nan=False) + profile: EmbeddingProfile + vector: tuple[float, ...] = Field(min_length=1) + + @field_validator("profile", mode="before") + @classmethod + def existing_profile_contract(cls, value): + if isinstance(value, dict) and set(value) - set(EmbeddingProfile.model_fields): + raise ValueError("query vector profile contains unsupported fields") # noqa: TRY003 + return value + + +class CustomArtifactSearchRequest(ArtifactSearchQuery): + query: str | None = Field(default=None, min_length=1, max_length=8192) + mode: Literal["text", "vector", "hybrid"] | None = None + query_vector: ArtifactQueryVector | None = None + indexes: dict[Literal["text", "vector"], tuple[str, ...]] | None = None + min_score: float = Field(default=0, ge=0, le=1, allow_inf_nan=False) + fusion: FusionSelection | None = None + + +@dataclass(frozen=True) +class CustomArtifactSearchOutcome: + matches: tuple[ArtifactSearchMatch, ...] + artifacts: tuple[Any, ...] + + +class CustomArtifactSearcher: + request_type = CustomArtifactSearchRequest + + def __init__(self, database, family, embedding_model=None): + self.database, self.family, self.embedding_model = database, family, embedding_model + self.artifact_type = type( + "CustomSearchArtifact", (Artifact[RootModel[dict[str, JsonValue]]],), {"family": family} + ) + + async def _configuration(self, connection, scope, request, *, config=None): # noqa: C901 - Validate one target or empty-scope snapshot. + durable = ( + ( + await connection.execute( + select(SCHEMAS.c.declaration) + .where(SCHEMAS.c.family == self.family) + .order_by(SCHEMAS.c.schema_version) + ) + ) + .scalars() + .all() + ) + if not durable: + raise ArtifactSearchFamilyNotFound(self.family) + declarations = [declaration for row in durable if (declaration := search_declaration(row)) is not None] + if not declarations: + raise ArtifactSearchUnsupported(self.family, field="family") + state = ( + (await connection.execute(select(STATES).where(key(STATES, scope, self.family)))).mappings().one_or_none() + ) + manifest = {} if state is None else state["manifest"] + if config is None and not manifest.get("config"): + binding = ARTIFACT_PROCESSING_BINDING_STATES_TABLE + definition = await connection.scalar( + select(binding.c.plugin_spec).where(binding.c.plugin_family == self.family) + ) + default = None if definition is None else search_declaration(definition["declaration"]) + config = default or declarations[0] + elif config is None: + # The manifest owns build identity/progress; query policy comes from the + # durable declaration in the same read snapshot, including during repair. + config = next( + (search_declaration(row) for row in durable if row["schema_version"] == manifest["default_schema"]), + None, + ) + if config is None: + raise ArtifactSearchUnavailable(self.family, "index declaration unavailable") + mode = request.mode or config.default_mode + self._validate_input(request, mode) + if mode not in config.modes: + raise ArtifactSearchUnsupported(self.family, field="mode") + try: + chosen = config.select_indexes(mode, request.indexes) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="indexes") from error + profiles = [index.profile for index in chosen if isinstance(index, VectorIndex)] + if profiles and any(profile != profiles[0] for profile in profiles): + raise ArtifactSearchUnsupported(self.family, field="indexes") + profile = profiles[0] if profiles else None + if request.query_vector is not None: + if request.query_vector.profile != profile: + raise ArtifactSearchUnsupported(self.family, field="query_vector.profile") + try: + vector_bytes(request.query_vector.vector, profile) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="query_vector.vector") from error + if request.fusion is not None: + try: + params = parse_fusion_parameters(request.fusion) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="fusion") from error + weights = {"text": config.text_weight, "vector": config.vector_weight} + if set(params.weights) - set(weights): + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + weights.update(params.weights) + if sum(weights.values()) <= 0: + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + if state is None: + has_content = await connection.scalar( + select(ARTIFACT_HEADS_TABLE.c.artifact_id) + .where(ARTIFACT_HEADS_TABLE.c.scope_id == scope, ARTIFACT_HEADS_TABLE.c.family == self.family) + .limit(1) + ) + if mode != "text" or has_content is not None: + raise ArtifactSearchUnavailable(self.family, "index has not been built") + return None, config, mode, (), None + if state["state"] != "ready": + raise ArtifactSearchUnavailable(self.family, "index rebuild is incomplete") + selected = await self._selected_indexes(connection, state, chosen) + if any(isinstance(index, VectorIndex) and index.profile != profile for _, _, index in selected): + raise ArtifactSearchUnsupported(self.family, field="indexes") + return state, config, mode, selected, profile + + async def _selected_indexes(self, connection, state, chosen): + kinds = {index.name: index.kind for index in chosen} + selected = [] + for _, entry in entries(state["manifest"]): + definition = dict( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == entry["index_def_id"]) + ) + ) + .mappings() + .one() + ) + if definition["index_name"] not in kinds: + continue + await projection_table(connection, definition) + index = parse_index(definition) + if index.kind != kinds[index.name]: + raise ArtifactSearchUnsupported(self.family, field="indexes") + selected.append((state, definition, index)) + if not set(kinds).issubset({definition["index_name"] for _, definition, _ in selected}): + raise ArtifactSearchUnavailable(self.family, "configuration lacks a selected index") + return selected + + async def search(self, scope_id, request, /, *, execution_context=None): + try: + return await self._search(scope_id, request, execution_context=execution_context) + except SQLAlchemyError as error: + raise ArtifactSearchUnavailable(self.family, "search storage is unavailable") from error + + async def _search(self, scope_id, request, /, *, execution_context=None): + execution = execution_context or ArtifactSearchExecutionContext() + access = execution.access + # Pin query policy across encoding outside the transaction; recheck build and access below. + async with AsyncExitStack() as resources: + if access is not None: + await resources.enter_async_context(access.defer_decision_audit()) + async with self.database.transaction(consistent_snapshot=True) as connection: + if access is not None: + await access.for_read(connection).require( + execution.principal, + AccessAction.SCOPE_READ, + ResourceRef.scope(scope_id), + context=execution.audit, + ) + state, config, mode, selected, profile = await self._configuration(connection, scope_id, request) + if state is None: + return CustomArtifactSearchOutcome((), ()) + query_vector = await self._encode_query(request, profile) + async with self.database.transaction(consistent_snapshot=True) as connection: + current, config, mode, selected, checked_profile = await self._configuration( + connection, scope_id, request, config=config + ) + if current["build_id"] != state["build_id"] or checked_profile != profile: + raise ArtifactSearchUnavailable(self.family, "search build changed during query preparation; retry") + bound = None if access is None else access.for_read(connection) + if bound is not None: + await bound.require( + execution.principal, + AccessAction.SCOPE_READ, + ResourceRef.scope(scope_id), + context=execution.audit, + ) + channels = await self._candidates( + connection, scope_id, request, mode, selected, query_vector, profile, bound, execution + ) + native_text = connection.dialect.name != "sqlite" + text_metric = "oceanbase_match" if native_text else "sqlite_bm25" + scored = self._rank(channels, config, mode, request, native_text, text_metric) + return await self._outcome(connection, scope_id, request, channels, scored, native_text, text_metric) + + async def _encode_query(self, request, profile): + query_vector = None + if profile is not None: + if request.query_vector is not None: + try: + query_vector = vector_bytes(request.query_vector.vector, profile) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="query_vector.vector") from error + else: + model = self.embedding_model + if model is None or model.profile != profile: + raise ArtifactSearchUnavailable(self.family, "query encoder unavailable for declared profile") + try: + result = await embed_query(model, (request.query,)) + query_vector = vector_bytes(result.vectors[0], profile) + except Exception as error: + raise ArtifactSearchUnavailable(self.family, "query encoder failed for declared profile") from error + if len(result.vectors) != 1: + raise ArtifactSearchUnavailable(self.family, "query encoder returned an invalid vector count") + return query_vector + + async def _candidates(self, connection, scope_id, request, mode, selected, query_vector, profile, bound, execution): + visible = {} + channels = {name: {} for name in ({"text", "vector"} if mode == "hybrid" else {mode})} + for state, definition, index in selected: + if index.kind == "text": + rows = await self._text( + connection, scope_id, state["build_id"], definition["index_def_id"], request.query + ) + else: + rows = await self._vector(connection, scope_id, definition, state["build_id"], query_vector, profile) + for identity, revision, raw in rows: + if identity not in visible: + visible[identity] = ( + bound is None + or ( + await bound.check( + execution.principal, + AccessAction.ARTIFACT_READ, + ResourceRef.artifact(scope_id, family=self.family, artifact_id=identity), + context=execution.audit, + ) + ).allowed + ) + if not visible[identity]: + continue + if not math.isfinite(raw): + raise ArtifactSearchUnavailable(self.family, "index returned a nonfinite score") + address = (identity, revision) + previous = channels[index.kind].get(address) + if previous is None or raw < previous: + channels[index.kind][address] = raw + return channels + + def _rank(self, channels, config, mode, request, native_text, text_metric): + budget = max(request.limit, config.candidate_limit) + ordered = { + name: sorted(values, key=lambda address: (values[address], address))[:budget] + for name, values in channels.items() + } + if mode == "hybrid": + params = RrfParameters(rank_constant=config.rrf_k) + weights = {"text": config.text_weight, "vector": config.vector_weight} + if request.fusion is not None: + try: + params = parse_fusion_parameters(request.fusion) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="fusion") from error + if set(params.weights) - set(weights): + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + weights.update(params.weights) + if sum(weights.values()) <= 0: + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + fused = fuse_rrf( + tuple( + FusionChannel( + name, + weights[name], + tuple(FusionCandidate(identity, rank) for rank, identity in enumerate(identities, 1)), + ) + for name, identities in ordered.items() + ), + params, + tie_break=lambda identity: identity, + ) + scored = [(hit.key, hit.score) for hit in fused] + elif mode == "text": + scored = [ + (identity, lexical_search_score((-1 if native_text else 1) * channels[mode][identity], text_metric)[0]) + for identity in ordered[mode] + ] + else: + scored = [(identity, 1 / (1 + channels[mode][identity])) for identity in ordered[mode]] + return scored + + async def _outcome(self, connection, scope_id, request, channels, scored, native_text, text_metric): + repository = custom_artifact_repository(self.family) + matches = [] + final = [(identity, score) for identity, score in scored if score >= request.min_score][: request.limit] + artifacts = await repository.get_many( + connection, + scope_id, + tuple( + ArtifactRef(family=self.family, artifact_id=identity, revision=revision) + for (identity, revision), score in final + ), + ) + for ((identity, revision), score), artifact in zip(final, artifacts, strict=True): + raw_scores = { + name: ChannelScore( + (-1 if native_text and name == "text" else 1) * values[(identity, revision)], + text_metric if name == "text" else "l2_distance", + native_text and name == "text", + ) + for name, values in channels.items() + if (identity, revision) in values + } + matches.append( + ArtifactSearchMatch(artifact.as_ref(), score, raw_scores if request.include_scores else None) + ) + public_artifacts = tuple( + self.artifact_type( + artifact_id=artifact.artifact_id, + revision=artifact.revision, + content=artifact.content.content, + lineage=artifact.lineage, + ) + for artifact in artifacts + ) + return CustomArtifactSearchOutcome(tuple(matches), public_artifacts) + + def _validate_input(self, request, mode): + if mode in {"text", "hybrid"} and request.query is None: + raise ArtifactSearchUnsupported(self.family, field="query") + if mode == "text" and request.query_vector is not None: + raise ArtifactSearchUnsupported(self.family, field="query_vector") + if mode == "vector" and ((request.query is None) == (request.query_vector is None)): + raise ArtifactSearchUnsupported(self.family, field="query_vector") + if mode != "hybrid" and request.fusion is not None: + raise ArtifactSearchUnsupported(self.family, field="fusion") + + async def _text(self, connection, scope, build_id, index_def_id, query): + expression = fts_match_query(query) + if expression is None: + return [] + if connection.dialect.name != "sqlite": + result = await connection.execute( + text("""SELECT artifact_id, revision, + MATCH(searchable_text) AGAINST(:query) AS score FROM pc_artifact_search_text + WHERE scope_id=:scope AND family=:family AND build_id=:build_id AND index_def_id=:index_def_id + AND MATCH(searchable_text) AGAINST(:query) > 0 ORDER BY score DESC, artifact_id"""), + { + "query": analyze_fts_query(query), + "scope": scope, + "family": self.family, + "build_id": build_id, + "index_def_id": index_def_id, + }, + ) + return [(row.artifact_id, int(row.revision), -float(row.score)) for row in result] + result = await connection.execute( + text("""SELECT artifact_id, revision, bm25(pc_artifact_search_fts) AS score + FROM pc_artifact_search_fts WHERE pc_artifact_search_fts MATCH :query + AND scope_id=:scope AND family=:family AND build_id=:build_id AND index_def_id=:index_def_id + ORDER BY score, artifact_id"""), + { + "query": expression, + "scope": scope, + "family": self.family, + "build_id": build_id, + "index_def_id": index_def_id, + }, + ) + return [(row.artifact_id, int(row.revision), float(row.score)) for row in result] + + async def _vector(self, connection, scope, definition, build_id, vector, profile): + table = await projection_table(connection, definition) + function = "vec_distance_l2" if connection.dialect.name == "sqlite" else "l2_distance" + # Table identity comes only from a validated server-owned space, never the request. + statement = text(f"""SELECT artifact_id, revision, MIN({function}(embedding, :vector)) AS distance + FROM {table.name} WHERE scope_id=:scope AND family=:family AND build_id=:build_id AND space_id=:space AND index_def_id=:index_def_id + GROUP BY artifact_id, revision ORDER BY distance, artifact_id""") # noqa: S608 - Function and table names come only from validated server-owned metadata. + if connection.dialect.name != "sqlite": + vector = list(struct.unpack(f"<{profile.dimension}f", vector)) + statement = statement.bindparams(bindparam("vector", type_=table.c.embedding.type)) + result = await connection.execute( + statement, + { + "vector": vector, + "scope": scope, + "family": self.family, + "build_id": build_id, + "space": definition["space_id"], + "index_def_id": definition["index_def_id"], + }, + ) + return [(row.artifact_id, int(row.revision), float(row.distance)) for row in result] diff --git a/src/powercontext/builtin/tags.py b/src/powercontext/builtin/tags.py index ff1c9626f7..4be736d541 100644 --- a/src/powercontext/builtin/tags.py +++ b/src/powercontext/builtin/tags.py @@ -27,9 +27,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from powercontext.artifacts import ArtifactRef -from powercontext.builtin.records import ArtifactReadFamily, BaseAccessError, InvalidBaseAccessRequestError +from powercontext.builtin.records import BaseAccessError, BaseArtifactFamily, InvalidBaseAccessRequestError -TaggableArtifactFamily = ArtifactReadFamily +TaggableArtifactFamily = BaseArtifactFamily TagMatch = Literal["all", "any"] TagTargetType = Literal["artifact"] diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index dcb6b7c740..806fe5a469 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -57,6 +57,8 @@ ArtifactHandleBinding, ArtifactInventoryStatistics, ArtifactPage, + ArtifactPluginPage, + ArtifactPluginState, ArtifactPublication, ArtifactReadFamily, ArtifactReference, @@ -115,6 +117,7 @@ CaptureWorkingNoteRequest, CatalogChangeProposal, CatalogTagResult, + ChangeArtifactPluginStateRequest, ClearScopeBindingRequest, ClearScopeBindingResponse, CodeQueryRequest, @@ -154,6 +157,7 @@ CreateTopicMemoryArtifactRequest, CreateWorkContractRequest, CurrentWorkHandoff, + CustomArtifactFamilySchema, DownloadRemoteSkillPackageRequest, DreamBudget, DreamCandidateRef, @@ -171,6 +175,8 @@ DreamSourceReference, DreamStatus, DreamUsage, + EnableAllArtifactPluginsResponse, + EnableArtifactPluginResult, EnrollRemoteSkillTargetRequest, EntryChange, EntryChangeOperation, @@ -341,6 +347,7 @@ RecurrenceStreak, RegisterSourceDefinitionRequest, RejectCandidateRequest, + ReloadArtifactPluginRequest, RemainingWorkItem, RememberMemoryRequest, RemoteAgentKind, @@ -520,6 +527,8 @@ "ArtifactHandleBinding", "ArtifactInventoryStatistics", "ArtifactPage", + "ArtifactPluginPage", + "ArtifactPluginState", "ArtifactPublication", "ArtifactReadFamily", "ArtifactReference", @@ -578,6 +587,7 @@ "CaptureWorkingNoteRequest", "CatalogChangeProposal", "CatalogTagResult", + "ChangeArtifactPluginStateRequest", "ClearScopeBindingRequest", "ClearScopeBindingResponse", "CodeQueryRequest", @@ -617,6 +627,7 @@ "CreateTopicMemoryArtifactRequest", "CreateWorkContractRequest", "CurrentWorkHandoff", + "CustomArtifactFamilySchema", "DownloadRemoteSkillPackageRequest", "DreamBudget", "DreamCandidateRef", @@ -634,6 +645,8 @@ "DreamSourceReference", "DreamStatus", "DreamUsage", + "EnableAllArtifactPluginsResponse", + "EnableArtifactPluginResult", "EnrollRemoteSkillTargetRequest", "EntryChange", "EntryChangeOperation", @@ -804,6 +817,7 @@ "RecurrenceStreak", "RegisterSourceDefinitionRequest", "RejectCandidateRequest", + "ReloadArtifactPluginRequest", "RemainingWorkItem", "RememberMemoryRequest", "RemoteAgentKind", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 6ad15c058a..0d7440c332 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -680,6 +680,109 @@ class ArtifactDreamingOperationCapability(BaseModel): effect: Effect +class CustomArtifactFamilySchema(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + family: StrictStr + schema_version: StrictStr + content_schema: dict[str, Any] + search_fields: Annotated[ + list[StrictStr], Field(description="Legacy persisted text fields; new plugins use search.") + ] + search: Annotated[ + dict[str, Any] | None, Field(description="Persistent declarative indexes and supported search modes.") + ] = None + + +class ReloadArtifactPluginRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + expected_generation: Annotated[StrictInt, Field(ge=0)] + + +class ChangeArtifactPluginStateRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + expected_generation: Annotated[StrictInt, Field(ge=1)] + + +class DesiredState(StrEnum): + ENABLED = "enabled" + DISABLED = "disabled" + UNLOADED = "unloaded" + + +class Phase(StrEnum): + ACTIVATING = "activating" + ENABLED = "enabled" + STOPPING = "stopping" + DISABLED = "disabled" + UNLOADING = "unloading" + UNLOADED = "unloaded" + FAILED = "failed" + + +class ArtifactPluginState(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + binding_name: StrictStr + plugin_id: StrictStr + family: StrictStr + binding_generation: Annotated[StrictInt, Field(ge=1)] + desired_state: DesiredState + phase: Phase + last_error: Annotated[StrictStr | None, Field(...)] + + +class ArtifactPluginPage(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + items: list[ArtifactPluginState] + + +class Outcome(StrEnum): + ACCEPTED = "accepted" + SKIPPED = "skipped" + FAILED = "failed" + + +class ErrorEnum(StrEnum): + CONFLICT = "conflict" + INVALID_PLUGIN = "invalid_plugin" + UNAVAILABLE = "unavailable" + + +class Error(RootModel[ErrorEnum | None]): + root: Annotated[ + ErrorEnum | None, Field(description="Safe per-plugin failure code; null for accepted or skipped entries.") + ] = None + + +class EnableArtifactPluginResult(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + binding_name: StrictStr + outcome: Outcome + status_url: Annotated[StrictStr, Field(description="Registration status URL for the asynchronous result.")] + state: ArtifactPluginState + error: Annotated[ + Error | None, Field(description="Safe per-plugin failure code; null for accepted or skipped entries.") + ] + + +class EnableAllArtifactPluginsResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + items: list[EnableArtifactPluginResult] + + class Configuration(StrEnum): CONFIGURED = "configured" UNCONFIGURED = "unconfigured" @@ -1630,7 +1733,7 @@ class Validation(StrEnum): UNKNOWN = "unknown" -class Outcome(StrEnum): +class Outcome1(StrEnum): SUCCESS = "success" FAILURE = "failure" UNKNOWN = "unknown" @@ -2122,7 +2225,7 @@ class CodeQueryResponse(RootModel[CodeQueryResult | CodeStatus]): root: CodeQueryResult | CodeStatus -class Outcome1(StrEnum): +class Outcome2(StrEnum): INJECTED = "injected" FAILED = "failed" @@ -2133,7 +2236,7 @@ class RecordBootstrapDeliveryRequest(BaseModel): ) scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] receipt_id: Annotated[StrictStr, Field(max_length=64, min_length=1, pattern="^[\\x21-\\x7E]+$")] - outcome: Outcome1 + outcome: Outcome2 class ContextAssemblyFamily(StrEnum): @@ -2249,11 +2352,57 @@ def _reject_invalid_search_value(cls, value: Any) -> Any: return value +class Distance(StrEnum): + L2 = "l2" + + +class Normalization(StrEnum): + NONE = "none" + UNIT = "unit" + + +class EmbeddingProfile(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + profile_id: Annotated[StrictStr, Field(min_length=1, pattern=".*\\S.*")] + model: Annotated[StrictStr, Field(min_length=1, pattern=".*\\S.*")] + dimension: Annotated[StrictInt, Field(ge=1)] + distance: Distance = Distance.L2 + normalization: Normalization = Normalization.UNIT + + +class QueryVector(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + profile: EmbeddingProfile + vector: Annotated[list[StrictFloat], Field(min_length=1)] + + +class TextItem(RootModel[StrictStr]): + root: Annotated[StrictStr, Field(min_length=1)] + + +class VectorItem(RootModel[StrictStr]): + root: Annotated[StrictStr, Field(min_length=1)] + + +class Indexes(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + text: Annotated[list[TextItem] | None, Field(min_length=1)] = None + vector: Annotated[list[VectorItem] | None, Field(min_length=1)] = None + + class SearchArtifactsRequest(BaseModel): model_config = ConfigDict( extra="forbid", ) - query: Annotated[StrictStr, Field(max_length=8192, min_length=1, pattern=".*\\S.*")] + query: Annotated[StrictStr | None, Field(max_length=8192, min_length=1, pattern=".*\\S.*")] = None + query_vector: QueryVector | None = None + indexes: Indexes | None = None limit: Annotated[StrictInt, Field(ge=1, le=200)] = 10 mode: Annotated[StrictStr | None, Field(min_length=1, pattern=".*\\S.*")] = None filters: dict[str, Any] | None = None @@ -2902,15 +3051,8 @@ class BaseArtifactFamily(StrEnum): TOPIC_MEMORY = "topic-memory" -class ArtifactReadFamily(StrEnum): - MEMORY = "memory" - ATOMIC_MEMORY = "atomic-memory" - EXPERIENCE = "experience" - SKILL = "skill" - HANDOFF = "handoff" - PROFILE = "profile" - PROMPT = "prompt" - TOPIC_MEMORY = "topic-memory" +class ArtifactReadFamily(RootModel[StrictStr]): + root: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[a-z][a-z0-9-]*$")] class StatsPeriod(StrEnum): @@ -3745,6 +3887,9 @@ class ArtifactCollectionItem(BaseModel): scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] family: ArtifactReadFamily artifact_id: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[\\x21-\\x7E]+$")] + schema_version: Annotated[ + StrictStr | None, Field(description="Persisted schema version for a custom Artifact family.") + ] = None revision: Annotated[StrictInt, Field(ge=1)] sources: list[SourceTypeReference] artifacts: list[ArtifactReference] @@ -3789,6 +3934,9 @@ class ArtifactRevision(BaseModel): scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] family: ArtifactReadFamily artifact_id: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[\\x21-\\x7E]+$")] + schema_version: Annotated[ + StrictStr | None, Field(description="Persisted schema version for a custom Artifact family.") + ] = None revision: Annotated[StrictInt, Field(ge=1)] content: dict[str, Any] sources: list[SourceTypeReference] @@ -4165,7 +4313,7 @@ class RecordSkillUsageRequest(BaseModel): selected: StrictBool invoked: Invoked validation: Validation - outcome: Outcome + outcome: Outcome1 task_source: SourceReference | None = None environment_fingerprint: Annotated[StrictStr | None, Field(pattern="^sha256:[0-9a-f]{64}$")] = None @@ -4886,6 +5034,7 @@ class Capabilities(BaseModel): model_config = ConfigDict( extra="forbid", ) + custom_artifact_schemas: Annotated[list[CustomArtifactFamilySchema], Field(validate_default=True)] = [] prompts: Annotated[dict[str, PromptCapability], Field(validate_default=True)] = {} artifact_dreaming: Annotated[ StrictBool, Field(description="Whether asynchronous Artifact Dream execution is configured.") diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index 9f817649fc..6f1e310386 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -22,6 +22,8 @@ ArtifactCreated, ArtifactHandleBinding, ArtifactPage, + ArtifactPluginPage, + ArtifactPluginState, ArtifactPublication, ArtifactRevision, ArtifactRevisionPage, @@ -45,6 +47,7 @@ CaptureContentSourceRequest, CaptureContentSourceResponse, CaptureWorkingNoteRequest, + ChangeArtifactPluginStateRequest, ClearScopeBindingRequest, ClearScopeBindingResponse, CodeQueryRequest, @@ -66,6 +69,7 @@ DownloadRemoteSkillPackageRequest, DreamRun, DreamRunPage, + EnableAllArtifactPluginsResponse, EnrollRemoteSkillTargetRequest, ExperienceArtifact, ExternalSkillResolution, @@ -150,6 +154,7 @@ RecordTaskOutcomeRequest, RegisterSourceDefinitionRequest, RejectCandidateRequest, + ReloadArtifactPluginRequest, RememberMemoryRequest, RemoteSkillPublication, RemoteSkillReceiptResponse, @@ -622,6 +627,181 @@ class AccessRequirement(BaseModel): access=AccessRequirement(action="server.observe", resource="server", scope_id_field=None, resolver="static"), ) +LIST_ARTIFACT_PLUGINS = Operation[None, ArtifactPluginPage]( + method="GET", + path="/v1/admin/artifact-plugins", + operation_id="list_artifact_plugins", + request_type=None, + request_location=None, + path_parameters=(), + response_type=ArtifactPluginPage, + success_status=200, + summary="List Artifact plugin registrations", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 200: {"description": "Durable registration states."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +GET_ARTIFACT_PLUGIN = Operation[None, ArtifactPluginState]( + method="GET", + path="/v1/admin/artifact-plugins/{binding_name}", + operation_id="get_artifact_plugin", + request_type=None, + request_location=None, + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=200, + summary="Get an Artifact plugin registration", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 200: {"description": "Durable registration state without plugin configuration secrets."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +ENABLE_ALL_ARTIFACT_PLUGINS = Operation[None, EnableAllArtifactPluginsResponse]( + method="POST", + path="/v1/admin/artifact-plugins/enable-all", + operation_id="enable_all_artifact_plugins", + request_type=None, + request_location=None, + path_parameters=(), + response_type=EnableAllArtifactPluginsResponse, + success_status=200, + summary="Enable the currently disabled Artifact plugins", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 200: {"description": "Per-plugin acceptance, skip or failure results."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +RELOAD_ARTIFACT_PLUGIN = Operation[ReloadArtifactPluginRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/reload", + operation_id="reload_artifact_plugin", + request_type=ReloadArtifactPluginRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Load or update a Worker-loaded Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Definition saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +ENABLE_ARTIFACT_PLUGIN = Operation[ChangeArtifactPluginStateRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/enable", + operation_id="enable_artifact_plugin", + request_type=ChangeArtifactPluginStateRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Enable an Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Target saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +DISABLE_ARTIFACT_PLUGIN = Operation[ChangeArtifactPluginStateRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/disable", + operation_id="disable_artifact_plugin", + request_type=ChangeArtifactPluginStateRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Disable an Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Target saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +UNLOAD_ARTIFACT_PLUGIN = Operation[ChangeArtifactPluginStateRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/unload", + operation_id="unload_artifact_plugin", + request_type=ChangeArtifactPluginStateRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Unload an Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Target saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + GET_CAPABILITIES = Operation[None, Capabilities]( method="GET", path="/v1/capabilities", diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index a9161ac59a..08971817f3 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -459,6 +459,319 @@ "x-powercontext-access": {"action": "server.observe", "resource": {"type": "server"}}, } }, + "/v1/admin/artifact-plugins": { + "get": { + "tags": ["artifact-plugins"], + "summary": "List Artifact plugin registrations", + "operationId": "list_artifact_plugins", + "responses": { + "200": { + "description": "Durable registration states.", + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginPage"}} + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + } + }, + "/v1/admin/artifact-plugins/{binding_name}": { + "get": { + "tags": ["artifact-plugins"], + "summary": "Get an Artifact plugin registration", + "operationId": "get_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "responses": { + "200": { + "description": "Durable registration state without plugin configuration secrets.", + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/enable-all": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Enable the currently disabled Artifact plugins", + "description": "Captures " + "registrations and " + "generations at " + "request start, then " + "accepts each " + "eligible enable " + "independently. " + "Enabled, unloaded, " + "failed and " + "transitioning " + "plugins are skipped. " + "Concurrent changes " + "and validation " + "failures are " + "reported per plugin " + "without rolling back " + "successes. Does not " + "discover or reload " + "plugins; poll each " + "status URL for the " + "asynchronous " + "outcome.", + "operationId": "enable_all_artifact_plugins", + "responses": { + "200": { + "description": "Per-plugin acceptance, skip or failure results.", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/EnableAllArtifactPluginsResponse"} + } + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/reload": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Load or update a Worker-loaded Artifact plugin", + "description": "Reads the " + "binding's " + "direct " + "child " + "entry " + "under " + "POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR " + "and fixes " + "its " + "resolved " + "immutable " + "directory. " + "The " + "manifest " + "binding_name " + "must " + "match the " + "directory " + "entry and " + "request " + "path. " + "Saves " + "validated " + "configuration " + "and " + "declarations " + "without " + "creating " + "a " + "pipeline. " + "Existing " + "enabled " + "or " + "disabled " + "plugins " + "keep " + "their " + "state; " + "initial, " + "unloaded " + "or failed " + "registrations " + "become " + "disabled. " + "This " + "initial " + "path " + "supports " + "the " + "global " + "SQLite " + "Supervisor. " + "Plugin " + "code is " + "loaded " + "only in " + "child " + "processes.", + "operationId": "reload_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ReloadArtifactPluginRequest"}} + }, + }, + "responses": { + "202": { + "description": "Definition saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/enable": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Enable an Artifact plugin", + "operationId": "enable_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ChangeArtifactPluginStateRequest"} + } + }, + }, + "responses": { + "202": { + "description": "Target saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/disable": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Disable an Artifact plugin", + "operationId": "disable_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ChangeArtifactPluginStateRequest"} + } + }, + }, + "responses": { + "202": { + "description": "Target saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/unload": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Unload an Artifact plugin", + "operationId": "unload_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ChangeArtifactPluginStateRequest"} + } + }, + }, + "responses": { + "202": { + "description": "Target saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, "/v1/capabilities": { "get": { "tags": ["capabilities"], @@ -6075,6 +6388,11 @@ "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, "family": {"$ref": "#/components/schemas/ArtifactReadFamily"}, "artifact_id": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": "^[\\x21-\\x7E]+$"}, + "schema_version": { + "type": "string", + "description": "Persisted schema version for a custom Artifact family.", + "nullable": True, + }, "revision": {"type": "integer", "minimum": 1.0}, "sources": {"items": {"$ref": "#/components/schemas/SourceTypeReference"}, "type": "array"}, "artifacts": {"items": {"$ref": "#/components/schemas/ArtifactReference"}, "type": "array"}, @@ -6143,6 +6461,11 @@ "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, "family": {"$ref": "#/components/schemas/ArtifactReadFamily"}, "artifact_id": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": "^[\\x21-\\x7E]+$"}, + "schema_version": { + "type": "string", + "description": "Persisted schema version for a custom Artifact family.", + "nullable": True, + }, "revision": {"type": "integer", "minimum": 1.0}, "content": {"additionalProperties": True, "type": "object"}, "sources": {"items": {"$ref": "#/components/schemas/SourceTypeReference"}, "type": "array"}, @@ -6875,8 +7198,106 @@ "type": "object", "required": ["operation", "output_kind", "effect"], }, + "CustomArtifactFamilySchema": { + "properties": { + "family": {"type": "string"}, + "schema_version": {"type": "string"}, + "content_schema": {"additionalProperties": True, "type": "object"}, + "search_fields": { + "items": {"type": "string"}, + "type": "array", + "description": "Legacy persisted text fields; new plugins use search.", + }, + "search": { + "additionalProperties": True, + "type": "object", + "description": "Persistent declarative indexes and supported search modes.", + }, + }, + "additionalProperties": False, + "type": "object", + "required": ["family", "schema_version", "content_schema", "search_fields"], + }, + "ReloadArtifactPluginRequest": { + "properties": {"expected_generation": {"type": "integer", "minimum": 0.0}}, + "additionalProperties": False, + "type": "object", + "required": ["expected_generation"], + }, + "ChangeArtifactPluginStateRequest": { + "properties": {"expected_generation": {"type": "integer", "minimum": 1.0}}, + "additionalProperties": False, + "type": "object", + "required": ["expected_generation"], + }, + "ArtifactPluginState": { + "properties": { + "binding_name": {"type": "string"}, + "plugin_id": {"type": "string"}, + "family": {"type": "string"}, + "binding_generation": {"type": "integer", "minimum": 1.0}, + "desired_state": {"type": "string", "enum": ["enabled", "disabled", "unloaded"]}, + "phase": { + "type": "string", + "enum": ["activating", "enabled", "stopping", "disabled", "unloading", "unloaded", "failed"], + }, + "last_error": {"type": "string", "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": [ + "binding_name", + "plugin_id", + "family", + "binding_generation", + "desired_state", + "phase", + "last_error", + ], + }, + "ArtifactPluginPage": { + "properties": { + "items": {"items": {"$ref": "#/components/schemas/ArtifactPluginState"}, "type": "array"} + }, + "additionalProperties": False, + "type": "object", + "required": ["items"], + }, + "EnableArtifactPluginResult": { + "properties": { + "binding_name": {"type": "string"}, + "outcome": {"type": "string", "enum": ["accepted", "skipped", "failed"]}, + "status_url": { + "type": "string", + "description": "Registration status URL for the asynchronous result.", + }, + "state": {"$ref": "#/components/schemas/ArtifactPluginState"}, + "error": { + "type": "string", + "enum": ["conflict", "invalid_plugin", "unavailable", None], + "description": "Safe per-plugin failure code; null for accepted or skipped entries.", + "nullable": True, + }, + }, + "additionalProperties": False, + "type": "object", + "required": ["binding_name", "outcome", "status_url", "state", "error"], + }, + "EnableAllArtifactPluginsResponse": { + "properties": { + "items": {"items": {"$ref": "#/components/schemas/EnableArtifactPluginResult"}, "type": "array"} + }, + "additionalProperties": False, + "type": "object", + "required": ["items"], + }, "Capabilities": { "properties": { + "custom_artifact_schemas": { + "items": {"$ref": "#/components/schemas/CustomArtifactFamilySchema"}, + "type": "array", + "default": [], + }, "prompts": { "additionalProperties": {"$ref": "#/components/schemas/PromptCapability"}, "type": "object", @@ -10803,9 +11224,51 @@ "required": ["method"], "x-powercontext-artifact-search-validation": "fusion", }, + "EmbeddingProfile": { + "properties": { + "profile_id": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, + "model": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, + "dimension": {"type": "integer", "minimum": 1.0}, + "distance": {"type": "string", "enum": ["l2"], "default": "l2"}, + "normalization": {"type": "string", "enum": ["none", "unit"], "default": "unit"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["profile_id", "model", "dimension"], + "description": "Existing embedding space identity " + "shared with Memory; no additional " + "modality or version fields.", + }, "SearchArtifactsRequest": { "properties": { "query": {"type": "string", "maxLength": 8192, "minLength": 1, "pattern": ".*\\S.*"}, + "query_vector": { + "properties": { + "profile": {"$ref": "#/components/schemas/EmbeddingProfile"}, + "vector": {"items": {"type": "number"}, "type": "array", "minItems": 1}, + }, + "additionalProperties": False, + "type": "object", + "required": ["profile", "vector"], + }, + "indexes": { + "properties": { + "text": { + "items": {"type": "string", "minLength": 1}, + "type": "array", + "minItems": 1, + "uniqueItems": True, + }, + "vector": { + "items": {"type": "string", "minLength": 1}, + "type": "array", + "minItems": 1, + "uniqueItems": True, + }, + }, + "additionalProperties": False, + "type": "object", + }, "limit": {"type": "integer", "maximum": 200.0, "minimum": 1.0, "default": 10}, "mode": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, "filters": {"additionalProperties": True, "type": "object"}, @@ -10817,7 +11280,6 @@ }, "additionalProperties": False, "type": "object", - "required": ["query"], "x-powercontext-artifact-search-validation": "request", }, "ArtifactChannelScore": { @@ -11803,19 +12265,7 @@ "topic-memory", ], }, - "ArtifactReadFamily": { - "type": "string", - "enum": [ - "memory", - "atomic-memory", - "experience", - "skill", - "handoff", - "profile", - "prompt", - "topic-memory", - ], - }, + "ArtifactReadFamily": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": "^[a-z][a-z0-9-]*$"}, "StatsPeriod": {"type": "string", "enum": ["today", "7d", "30d"]}, "CandidateFamily": { "type": "string", diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index fa165f2362..31ed7416bd 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -52,6 +52,7 @@ from powercontext.artifacts.search import ( ArtifactSearchExecutionContext, ArtifactSearchFamilyNotFound, + ArtifactSearchUnavailable, ArtifactSearchUnsupported, ) from powercontext.builtin.artifacts.atomic_memory.errors import ( @@ -416,6 +417,8 @@ ArtifactFamilyAccessCapability, ArtifactHandleBinding, ArtifactPage, + ArtifactPluginPage, + ArtifactPluginState, ArtifactReadFamily, ArtifactReference, ArtifactRevision, @@ -433,6 +436,7 @@ CaptureContentSourceRequest, CaptureContentSourceResponse, CaptureWorkingNoteRequest, + ChangeArtifactPluginStateRequest, ClearScopeBindingRequest, ClearScopeBindingResponse, CodeQueryRequest, @@ -455,6 +459,7 @@ DownloadRemoteSkillPackageRequest, DreamRun, DreamRunPage, + EnableAllArtifactPluginsResponse, EnrollRemoteSkillTargetRequest, ErrorDetail, ErrorResponse, @@ -536,6 +541,7 @@ RecordTaskOutcomeRequest, RegisterSourceDefinitionRequest, RejectCandidateRequest, + ReloadArtifactPluginRequest, RememberMemoryRequest, RemoteSkillAction, RemoteSkillPublication, @@ -697,8 +703,11 @@ CREATE_SOURCE, CREATE_SUBJECT_SOURCE, CREATE_WORK_CONTRACT, + DISABLE_ARTIFACT_PLUGIN, DOWNLOAD_REMOTE_SKILL_PACKAGE, DOWNLOAD_SKILL_PACKAGE, + ENABLE_ALL_ARTIFACT_PLUGINS, + ENABLE_ARTIFACT_PLUGIN, ENROLL_REMOTE_SKILL_TARGET, FINALIZE_HANDOFF, FLUSH_MEMORY, @@ -709,6 +718,7 @@ GENERATE_SKILL, GET_ACCESS_PRINCIPAL, GET_ARTIFACT, + GET_ARTIFACT_PLUGIN, GET_ARTIFACT_REVISION, GET_ARTIFACT_TAGS, GET_CANDIDATE, @@ -741,6 +751,7 @@ LIST_ACCESS_BINDINGS, LIST_ACCESS_RESOURCES, LIST_ACCESS_ROLES, + LIST_ARTIFACT_PLUGINS, LIST_ARTIFACT_REVISIONS, LIST_ARTIFACTS, LIST_CANDIDATES, @@ -772,6 +783,7 @@ RECORD_TASK_OUTCOME, REGISTER_SOURCE_DEFINITION, REJECT_CANDIDATE, + RELOAD_ARTIFACT_PLUGIN, REMEMBER_MEMORY, RENAME_REMOTE_SKILL_TARGET, REPLACE_ACCESS_BINDING, @@ -795,6 +807,7 @@ SET_DEFAULT_SCOPE, SET_SCOPE_BINDING, SUBMIT_SOURCE_OBSERVATION, + UNLOAD_ARTIFACT_PLUGIN, UNPUBLISH_REMOTE_SKILL, UPDATE_SCOPE, UPDATE_SKILL_LIFECYCLE, @@ -803,6 +816,7 @@ ) from powercontext.http._generated.schema import OPENAPI_SCHEMA from powercontext.server import mapping +from powercontext.server.artifact_plugins import ArtifactPluginManager from powercontext.server.atomic_memory import add_atomic_memory_routes from powercontext.server.atomic_memory import record_response as atomic_record_response from powercontext.server.authentication import AuthenticationProvider @@ -1360,6 +1374,7 @@ def create_app( middleware=list(middleware), ) app.openapi_version = OPENAPI_VERSION + app.state.artifact_plugin_manager = None app.state.application = application app.state.capability_provider = capability_provider app.state.readiness_probe = readiness_probe @@ -1457,6 +1472,13 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: _add_route(app, GET_READINESS, get_readiness) _add_route(app, GET_SERVER_INFO, get_server_info) _add_route(app, GET_CAPABILITIES, get_capabilities) + _add_route(app, LIST_ARTIFACT_PLUGINS, list_artifact_plugins) + _add_route(app, ENABLE_ALL_ARTIFACT_PLUGINS, enable_all_artifact_plugins) + _add_route(app, GET_ARTIFACT_PLUGIN, get_artifact_plugin) + _add_route(app, RELOAD_ARTIFACT_PLUGIN, reload_artifact_plugin) + _add_route(app, ENABLE_ARTIFACT_PLUGIN, enable_artifact_plugin) + _add_route(app, DISABLE_ARTIFACT_PLUGIN, disable_artifact_plugin) + _add_route(app, UNLOAD_ARTIFACT_PLUGIN, unload_artifact_plugin) _add_route(app, LIST_SCOPES, list_scopes) _add_route(app, CREATE_SCOPE, create_scope) _add_route(app, GET_DEFAULT_SCOPE, get_default_scope) @@ -1670,11 +1692,91 @@ async def access_readiness_checks(state: Any) -> dict[str, str]: } +_PluginBindingPath = Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")] + + +def _require_artifact_plugin_manager(request: Request) -> ArtifactPluginManager: + manager = request.app.state.artifact_plugin_manager + if manager is None: + raise _RuntimeNotReadyError + return manager + + +async def list_artifact_plugins(http_request: Request) -> ArtifactPluginPage: + return ArtifactPluginPage(items=await _require_artifact_plugin_manager(http_request).list_registrations()) + + +async def enable_all_artifact_plugins(http_request: Request) -> EnableAllArtifactPluginsResponse: + return await _require_artifact_plugin_manager(http_request).enable_all() + + +async def get_artifact_plugin(binding_name: _PluginBindingPath, http_request: Request) -> ArtifactPluginState: + return await _require_artifact_plugin_manager(http_request).get(binding_name) + + +async def reload_artifact_plugin( + binding_name: _PluginBindingPath, + request: ReloadArtifactPluginRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).reload(binding_name, request) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + +async def enable_artifact_plugin( + binding_name: _PluginBindingPath, + request: ChangeArtifactPluginStateRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).change( + binding_name, request.expected_generation, "enabled" + ) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + +async def disable_artifact_plugin( + binding_name: _PluginBindingPath, + request: ChangeArtifactPluginStateRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).change( + binding_name, request.expected_generation, "disabled" + ) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + +async def unload_artifact_plugin( + binding_name: _PluginBindingPath, + request: ChangeArtifactPluginStateRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).change( + binding_name, request.expected_generation, "unloaded" + ) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + async def get_capabilities(request: Request) -> Capabilities: capability_provider: CapabilityProvider | None = request.app.state.capability_provider - if capability_provider is not None: - return capability_provider() - return request.app.state.capabilities + value = capability_provider() if capability_provider is not None else request.app.state.capabilities + manager = request.app.state.artifact_plugin_manager + if manager is None: + return value + declarations = await manager.declarations() + return Capabilities.model_validate({ + **value.model_dump(mode="json"), + "artifact_families": value.artifact_families + + sorted({item["family"] for item in declarations} - set(value.artifact_families)), + "custom_artifact_schemas": declarations, + }) async def get_server_info(request: Request) -> ServerInfo: @@ -2618,12 +2720,12 @@ def _list_artifacts_query( async def list_artifacts( scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], request: Annotated[ListArtifactsRequest, Depends(_list_artifacts_query)], application: Annotated[ServerApplication, Depends(_require_application)], ) -> ArtifactPage: result = await application.records.for_scope(scope_id).query_artifacts( - family.value, + family, limit=request.limit, cursor=request.cursor, **( @@ -2739,13 +2841,13 @@ def _list_artifact_revisions_query( async def list_artifact_revisions( scope_id: _ScopePathId, - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], artifact_id: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$")], request: Annotated[ListArtifactRevisionsRequest, Depends(_list_artifact_revisions_query)], application: Annotated[ServerApplication, Depends(_require_application)], ) -> ArtifactRevisionPage: result = await application.records.for_scope(scope_id).list_artifact_revisions( - family.value, artifact_id, limit=request.limit, cursor=request.cursor + family, artifact_id, limit=request.limit, cursor=request.cursor ) return ArtifactRevisionPage( items=[_artifact_collection_item_response(item) for item in result.items], @@ -2900,7 +3002,7 @@ async def query_artifact_tags( async def get_artifact( scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], artifact_id: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$")], http_request: Request, response: Response, @@ -2911,11 +3013,11 @@ async def get_artifact( ] = None, ) -> ArtifactRevision | Response: result = await application.records.for_scope(scope_id).get_artifact( - family.value, + family, artifact_id, **( {"execution_context": _atomic_execution_context(http_request, GET_ARTIFACT.operation_id)} - if family.value == "atomic-memory" + if family == "atomic-memory" else {} ), ) @@ -2965,19 +3067,19 @@ async def replace_artifact( async def get_artifact_revision( scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], artifact_id: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$")], revision: Annotated[int, Path(ge=1)], http_request: Request, application: Annotated[ServerApplication, Depends(_require_application)], ) -> ArtifactRevision: result = await application.records.for_scope(scope_id).get_artifact_revision( - family.value, + family, artifact_id, revision, **( {"execution_context": _atomic_execution_context(http_request, GET_ARTIFACT_REVISION.operation_id)} - if family.value == "atomic-memory" + if family == "atomic-memory" else {} ), ) @@ -3005,6 +3107,7 @@ def _artifact_revision_response(value: RuntimeArtifactRecord) -> ArtifactRevisio return ArtifactRevision( scope_id=value.scope_id, family=ArtifactReadFamily(value.family), + schema_version=value.schema_version, artifact_id=value.artifact_id, revision=value.revision, content=value.content, @@ -3033,6 +3136,7 @@ def _artifact_collection_item_response(value: RuntimeArtifactCollectionItem) -> return ArtifactCollectionItem( scope_id=value.scope_id, family=ArtifactReadFamily(value.family), + schema_version=value.schema_version, artifact_id=value.artifact_id, revision=value.revision, sources=[mapping.source_type_reference(ref) for ref in value.sources], @@ -5185,7 +5289,7 @@ def _path_artifact_family(payload: Mapping[str, Any]) -> str: def _path_artifact_read_family(payload: Mapping[str, Any]) -> str: try: - return ArtifactReadFamily(_nested_request_value(payload, "family")).value + return ArtifactReadFamily(_nested_request_value(payload, "family")).root except ValueError as error: raise AccessInvalidRequestError("artifact-family") from error @@ -5196,7 +5300,16 @@ def _path_artifact_read_access( ) -> tuple[tuple[AccessAction, ResourceRef], ...]: if _path_artifact_read_family(payload) in {BaseArtifactFamily.MEMORY.value, "topic-memory"}: return _path_scope_access(payload, action=AccessAction.SCOPE_READ) - return _path_artifact_access(payload, action=AccessAction.ARTIFACT_READ) + return ( + ( + AccessAction.ARTIFACT_READ, + ResourceRef.artifact( + _nested_request_value(payload, "scope_id"), + family=_path_artifact_read_family(payload), + artifact_id=_nested_request_value(payload, "artifact_id"), + ), + ), + ) def _path_artifact_write_access( @@ -5645,6 +5758,13 @@ def _map_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None]: return 426, error.code, "Upgrade to a client supporting Dream contract 2.", {"required_dream_contract": 2} if isinstance(error, ArtifactSearchFamilyNotFound): return status.HTTP_404_NOT_FOUND, "artifact_family_not_found", "The Artifact Family was not found.", None + if isinstance(error, ArtifactSearchUnavailable): + return ( + status.HTTP_503_SERVICE_UNAVAILABLE, + "artifact_search_unavailable", + "The declared search index or resource is not ready.", + None, + ) if isinstance(error, ArtifactSearchUnsupported): return ( status.HTTP_422_UNPROCESSABLE_CONTENT, diff --git a/src/powercontext/server/artifact_plugins.py b/src/powercontext/server/artifact_plugins.py new file mode 100644 index 0000000000..bd453334df --- /dev/null +++ b/src/powercontext/server/artifact_plugins.py @@ -0,0 +1,616 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Server-owned hot registration and retained reads for custom Artifact plugins.""" + +from __future__ import annotations + +import asyncio +import logging +import re +from collections.abc import Callable +from contextlib import AsyncExitStack +from datetime import timedelta +from functools import partial +from pathlib import Path +from typing import Any + +from sqlalchemy import insert, select, union, update +from sqlalchemy.exc import IntegrityError, SQLAlchemyError + +from powercontext._logging import log_safely +from powercontext.builtin.artifact_plugin_validation import validate_plugin +from powercontext.builtin.dream.bindings import DREAM_BINDINGS +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.sources import SOURCE_PROCESSING_BINDINGS +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, + ARTIFACT_SEARCH_STATES_TABLE, + SCOPES_TABLE, +) +from powercontext.builtin.records import BaseValueConflictError, BaseValueNotFoundError, InvalidBaseAccessRequestError +from powercontext.builtin.runtime.artifact_plugins import ( + ArtifactPluginWorkerSpec, + run_artifact_plugin_worker, +) +from powercontext.builtin.runtime.artifact_processing import ( + ArtifactProcessingBinding, + ArtifactProcessingSupervisor, + ArtifactProcessingSupervisors, + SpawnArtifactProcessingWorkerLauncher, +) +from powercontext.builtin.runtime.config import DatabaseConfig +from powercontext.builtin.runtime.processing_discovery import SourceProcessingPendingProvider +from powercontext.http._generated.models import ( + ArtifactPluginState, + EnableAllArtifactPluginsResponse, + EnableArtifactPluginResult, + ReloadArtifactPluginRequest, +) +from powercontext.server.authz.profiles import register_custom_artifact_access_family + +_STABLE_PHASES = frozenset({"enabled", "disabled", "unloaded", "failed"}) +_STATES = ARTIFACT_PROCESSING_BINDING_STATES_TABLE +_STATE_COLUMNS = ( + _STATES.c.binding_name, + _STATES.c.plugin_id, + _STATES.c.plugin_family, + _STATES.c.binding_generation, + _STATES.c.desired_state, + _STATES.c.phase, + _STATES.c.last_error, +) +_SCHEMAS = ARTIFACT_FAMILY_SCHEMAS_TABLE +logger = logging.getLogger(__name__) + + +class ArtifactPluginManager: + """Own registrations and transitions; user code stays outside the Server process.""" + + def __init__( + self, + database: AsyncDatabase, + config: DatabaseConfig, + supervisor: ArtifactProcessingSupervisors | None, + security_builder: Callable[[], dict[str, Any] | None], + on_supervisor: Callable[[ArtifactProcessingSupervisors], None], + plugins_dir: Path | None = None, + embedding_model=None, + inference_config=None, + runtime_config=None, + ) -> None: + self.database = database + self.index_builder = ArtifactIndexBuilder(database, embedding_model) + self.inference_config = inference_config + self.runtime_config = runtime_config + self.config = config + self.supervisor = supervisor + self.security_builder = security_builder + self.on_supervisor = on_supervisor + self.plugins_dir = None if plugins_dir is None else plugins_dir.expanduser().resolve() + self._resources = AsyncExitStack() + self._supervisor_lock = asyncio.Lock() + self._locks: dict[str, asyncio.Lock] = {} + self._tasks: set[asyncio.Task[None]] = set() + self._index_locks: dict[str, asyncio.Lock] = {} + self._index_task: asyncio.Task[None] | None = None + self._index_failures: set[tuple[str, str]] = set() + self._index_refresh_families: set[str] = set() + + async def __aenter__(self): + async with self.database.transaction() as connection: + families = (await connection.execute(select(_SCHEMAS.c.family).distinct())).scalars().all() + for family in families: + register_custom_artifact_access_family(family) + rows = (await connection.execute(select(_STATES).where(_STATES.c.plugin_id.is_not(None)))).mappings().all() + restored = [] + for persisted in rows: + if persisted["phase"] in {"failed", "disabled", "unloaded"}: + continue + row = dict(persisted) + row["binding_generation"] += 1 + row["phase"] = {"enabled": "activating", "disabled": "stopping", "unloaded": "unloading"}[ + row["desired_state"] + ] + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == row["binding_name"]) + .values(binding_generation=row["binding_generation"], phase=row["phase"]) + ) + restored.append(row) + self._index_refresh_families.update(families) + for row in restored: + await self._transition(row) + await self._discover() + # Retained indexes recover in the background; encoding existing content + # must not delay Server startup or availability of unrelated families. + self._index_task = asyncio.create_task(self._maintain_search(), name="artifact-index-maintenance") + return self + + async def __aexit__(self, *_): + if self._index_task is not None: + self._index_task.cancel() + await asyncio.gather(self._index_task, return_exceptions=True) + try: + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + finally: + await self._resources.aclose() + + async def _ensure_supervisor(self) -> ArtifactProcessingSupervisors: + async with self._supervisor_lock: + if self.supervisor is None: + controller = await self._resources.enter_async_context( + ArtifactProcessingSupervisor( + database=self.database, + bindings=(), + lease_mode="single-process", + ) + ) + self.supervisor = ArtifactProcessingSupervisors((controller,)) + self.on_supervisor(self.supervisor) + return self.supervisor + + async def _row(self, name: str, *, include_definition: bool = False) -> dict[str, Any]: + columns = (*_STATE_COLUMNS, _STATES.c.plugin_spec) if include_definition else _STATE_COLUMNS + async with self.database.transaction() as connection: + row = ( + ( + await connection.execute( + select(*columns).where(_STATES.c.binding_name == name, _STATES.c.plugin_id.is_not(None)) + ) + ) + .mappings() + .first() + ) + if row is None: + raise BaseValueNotFoundError("artifact_plugin", name) + return dict(row) + + @staticmethod + def _state(row: dict[str, Any]) -> ArtifactPluginState: + return ArtifactPluginState.model_validate({ + "binding_name": row["binding_name"], + "plugin_id": row["plugin_id"], + "family": row["plugin_family"], + "binding_generation": row["binding_generation"], + "desired_state": row["desired_state"], + "phase": row["phase"], + "last_error": row["last_error"], + }) + + async def get(self, name: str) -> ArtifactPluginState: + return self._state(await self._row(name)) + + async def list_registrations(self) -> list[ArtifactPluginState]: + async with self.database.transaction() as connection: + rows = ( + ( + await connection.execute( + select(*_STATE_COLUMNS).where(_STATES.c.plugin_id.is_not(None)).order_by(_STATES.c.binding_name) + ) + ) + .mappings() + .all() + ) + return [self._state(dict(row)) for row in rows] + + async def declarations(self) -> list[dict[str, Any]]: + async with self.database.transaction() as connection: + return list( + ( + await connection.execute( + select(_SCHEMAS.c.declaration).order_by(_SCHEMAS.c.family, _SCHEMAS.c.schema_version) + ) + ).scalars() + ) + + async def _discover(self) -> None: + directory = self.plugins_dir + if directory is None: + return + try: + entries = await asyncio.to_thread(lambda: sorted(directory.iterdir())) + except OSError as error: + log_safely( + logger, logging.WARNING, f"Artifact plugin discovery directory unavailable ({type(error).__name__})" + ) + return + known = {state.binding_name for state in await self.list_registrations()} + for entry in entries: + if entry.name in known or not await asyncio.to_thread(entry.is_dir): + continue + try: + await self.reload(entry.name, ReloadArtifactPluginRequest(expected_generation=0)) + except Exception as error: + log_safely( + logger, + logging.WARNING, + f"Artifact plugin discovery failed: {entry.name!r} ({type(error).__name__})", + ) + + async def reload(self, name: str, request: ReloadArtifactPluginRequest) -> ArtifactPluginState: + if re.fullmatch(r"[a-z][a-z0-9-]{0,127}", name) is None: + raise InvalidBaseAccessRequestError("binding_name", "must be a canonical binding name") + if name in {*SOURCE_PROCESSING_BINDINGS, *DREAM_BINDINGS.values()}: + raise InvalidBaseAccessRequestError("binding_name", "is reserved for a built-in processor") + directory = self.plugins_dir + if directory is None: + raise InvalidBaseAccessRequestError("artifact_plugins_dir", "must be configured on the Server") + try: + path = await asyncio.to_thread(lambda: (directory / name).resolve(strict=True)) + is_directory = await asyncio.to_thread(path.is_dir) + except (OSError, RuntimeError) as error: + raise InvalidBaseAccessRequestError( + "binding_name", "must identify a deployed plugin directory entry" + ) from error + if not is_directory: + raise InvalidBaseAccessRequestError("binding_name", "must identify a deployed plugin directory entry") + try: + descriptor = await validate_plugin(str(path)) + except (ValueError, OSError, TimeoutError) as error: + raise InvalidBaseAccessRequestError("plugin", "declaration or configuration validation failed") from error + if descriptor["manifest"]["binding_name"] != name: + raise InvalidBaseAccessRequestError( + "binding_name", "must match plugin.yml, directory entry and request path" + ) + definition = {"directory": str(path), **descriptor} + return await self._accept(name, request.expected_generation, None, definition) + + async def enable_all(self) -> EnableAllArtifactPluginsResponse: + # Freeze eligibility and generations before any per-plugin validation can yield. + snapshot = await self.list_registrations() + results = [] + for state in snapshot: + outcome, error = "skipped", None + current = state + if state.phase == "disabled": + try: + current = await self.change( + state.binding_name, state.binding_generation, "enabled", require_disabled=True + ) + outcome = "accepted" + except Exception as failure: + outcome = "failed" + error = ( + "conflict" + if isinstance(failure, BaseValueConflictError) + else "invalid_plugin" + if isinstance(failure, InvalidBaseAccessRequestError) + else "unavailable" + ) + current = await self.get(state.binding_name) + results.append( + EnableArtifactPluginResult.model_validate({ + "binding_name": state.binding_name, + "outcome": outcome, + "status_url": f"/v1/admin/artifact-plugins/{state.binding_name}", + "state": current, + "error": error, + }) + ) + return EnableAllArtifactPluginsResponse(items=results) + + async def change( + self, name: str, expected: int, target: str, *, require_disabled: bool = False + ) -> ArtifactPluginState: + row = await self._row(name, include_definition=True) + if ( + row["binding_generation"] != expected + or row["phase"] not in _STABLE_PHASES + or (require_disabled and row["phase"] != "disabled") + ): + raise BaseValueConflictError("artifact_plugin", name) + definition = row["plugin_spec"] + if target == "enabled": + if definition is None or row["phase"] == "failed": + raise InvalidBaseAccessRequestError( + "plugin", "an unloaded or failed plugin requires reload before enabling" + ) + if isinstance(self.config, SQLiteConfig) and self.config.is_in_memory: + raise InvalidBaseAccessRequestError( + "database", "Artifact plugin generation requires persistent storage" + ) + try: + descriptor = await validate_plugin(definition["directory"], definition["manifest"]) + except (ValueError, OSError, TimeoutError) as error: + raise InvalidBaseAccessRequestError("plugin", "saved plugin validation failed") from error + if descriptor != {"manifest": definition["manifest"], "declaration": definition["declaration"]}: + raise InvalidBaseAccessRequestError("plugin", "changed code must be registered with reload") + return await self._accept(name, expected, target, definition, require_disabled=require_disabled) + + async def _validate_identity(self, connection, name, old, definition): # noqa: C901 - Validate durable ownership and immutable declarations together. + declaration = None if definition is None else definition["declaration"] + if declaration is None: + if old is None: + raise BaseValueNotFoundError("artifact_plugin", name) + plugin_id, family = old["plugin_id"], old["plugin_family"] + else: + plugin_id, family = declaration["plugin_id"], declaration["family"] + if old is not None and (old["plugin_id"], old["plugin_family"]) != (plugin_id, family): + raise BaseValueConflictError("artifact_plugin_identity", name) + owner = await connection.scalar( + select(_STATES.c.binding_name) + .where(_STATES.c.plugin_family == family, _STATES.c.binding_name != name) + .limit(1) + ) + if owner is not None: + raise BaseValueConflictError("artifact_family", family) + if declaration is not None: + scope_ids = definition["manifest"]["scope_ids"] + if scope_ids is not None: + scopes = set( + ( + await connection.execute( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id.in_(scope_ids)) + ) + ).scalars() + ) + if scopes != set(scope_ids): + raise InvalidBaseAccessRequestError("scope_ids", "must identify existing Scopes") + durable = {key: declaration[key] for key in ("family", "schema_version", "content_schema", "search_fields")} + if "search" in declaration: + durable["search"] = declaration["search"] + from powercontext.builtin.persistence.artifact_indexes import register_definition, search_declaration + + search = search_declaration(durable) + saved = ( + (await connection.execute(select(_SCHEMAS.c.declaration).where(_SCHEMAS.c.family == family))) + .scalars() + .all() + ) + # Only query policy may change; persisted index identity survives unload and restart. + query_policy = { + "modes", + "default_mode", + "defaults", + "candidate_limit", + "rrf_k", + "text_weight", + "vector_weight", + } + fixed = None if search is None else search.model_dump(exclude=query_policy) + for item in saved: + previous_search = search_declaration(item) + previous_fixed = None if previous_search is None else previous_search.model_dump(exclude=query_policy) + if previous_fixed != fixed: + raise BaseValueConflictError("artifact_search_declaration", family) + if ( + saved + and search is not None + and all(item["schema_version"] != durable["schema_version"] for item in saved) + ): + raise BaseValueConflictError("artifact_search_declaration", family) + identity = (_SCHEMAS.c.family == family) & (_SCHEMAS.c.schema_version == durable["schema_version"]) + previous = await connection.scalar(select(_SCHEMAS.c.declaration).where(identity)) + if previous is None: + await connection.execute( + insert(_SCHEMAS).values( + family=family, schema_version=durable["schema_version"], declaration=durable + ) + ) + elif previous["content_schema"] != durable["content_schema"]: + raise BaseValueConflictError("artifact_schema", (family, durable["schema_version"])) + elif previous != durable: + await connection.execute(update(_SCHEMAS).where(identity).values(declaration=durable)) + if search is not None: + for index in search.indexes: + try: + await register_definition(connection, family, durable["schema_version"], index) + except ValueError as error: + raise BaseValueConflictError( + "artifact_search_definition", (family, index.name, index.projection_version) + ) from error + return plugin_id, family + + async def _accept( # noqa: C901 - Registration CAS and lifecycle acceptance share one transaction. + self, + name: str, + expected: int, + target: str | None, + definition: dict[str, Any] | None, + *, + require_disabled: bool = False, + ) -> ArtifactPluginState: + lock = self._locks.setdefault(name, asyncio.Lock()) + if lock.locked(): + raise BaseValueConflictError("artifact_plugin", name) + async with lock: + try: + async with self.database.transaction() as connection: + # This is also the per-registration serialization point used by Worker commit guards. + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == name) + .values(binding_generation=_STATES.c.binding_generation) + ) + old = ( + (await connection.execute(select(_STATES).where(_STATES.c.binding_name == name))) + .mappings() + .first() + ) + generation = 0 if old is None else int(old["binding_generation"]) + if ( + generation != expected + or (old is not None and (old["plugin_id"] is None or old["phase"] not in _STABLE_PHASES)) + or (require_disabled and (old is None or old["phase"] != "disabled")) + ): + raise BaseValueConflictError("artifact_plugin", name) + if target is None: + target = "enabled" if old is not None and old["phase"] == "enabled" else "disabled" + if target == "enabled": + self.security_builder() + if ( + old is not None + and old["plugin_spec"] == definition + and old["desired_state"] == target + and old["phase"] == target + ): + return self._state(dict(old)) + plugin_id, family = await self._validate_identity(connection, name, old, definition) + values = { + "plugin_id": plugin_id, + "plugin_family": family, + "plugin_spec": definition, + "binding_generation": generation + 1, + "desired_state": target, + "phase": {"enabled": "activating", "disabled": "stopping", "unloaded": "unloading"}[target], + "last_error": None, + } + if old is None and target == "disabled": + values["phase"] = "disabled" + if old is None: + await connection.execute(insert(_STATES).values(binding_name=name, **values)) + else: + await connection.execute(update(_STATES).where(_STATES.c.binding_name == name).values(**values)) + except IntegrityError as error: + raise BaseValueConflictError("artifact_plugin", name) from error + register_custom_artifact_access_family(family) + if old is None: + self._index_refresh_families.add(family) + row = {"binding_name": name, **values} + if row["phase"] == target: + return self._state(row) + task = asyncio.create_task(self._transition(row), name=f"artifact-plugin-transition-{name}") + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return self._state(row) + + async def _transition(self, row: dict[str, Any]) -> None: + name, generation = row["binding_name"], row["binding_generation"] + try: + definition = row["plugin_spec"] + target = row["desired_state"] + supervisor = await self._ensure_supervisor() if target == "enabled" else self.supervisor + if supervisor is not None: + await supervisor.replace_binding(name, None) + if target == "enabled" and supervisor is not None: + descriptor = await validate_plugin(definition["directory"], definition["manifest"]) + if descriptor != {"manifest": definition["manifest"], "declaration": definition["declaration"]}: + raise ValueError("plugin code differs from the accepted definition") # noqa: TRY003, TRY301 + manifest, declaration = descriptor["manifest"], descriptor["declaration"] + spec = ArtifactPluginWorkerSpec( + self.config, + name, + generation, + definition, + self.security_builder(), + self.inference_config, + self.runtime_config, + ) + + async def scopes(connection, candidates): + phase = await connection.scalar( + select(_STATES.c.phase).where( + _STATES.c.binding_name == name, _STATES.c.binding_generation == generation + ) + ) + if phase != "enabled": + return frozenset() + scope_ids = manifest["scope_ids"] + return frozenset(candidates if scope_ids is None else set(candidates).intersection(scope_ids)) + + await supervisor.replace_binding( + name, + ArtifactProcessingBinding( + binding_name=name, + artifact_family=declaration["family"], + launcher=SpawnArtifactProcessingWorkerLauncher(partial(run_artifact_plugin_worker, spec)), + max_workers=manifest["processing"]["max_workers"], + worker_timeout_seconds=manifest["processing"]["timeout_seconds"], + automatic_processing_interval=timedelta(seconds=manifest["processing"]["interval_seconds"]), + pending_provider=SourceProcessingPendingProvider(self.database, name, declaration["family"]), + automatic_scope_filter=scopes, + invocation_scope_filter=scopes, + ), + ) + async with self.database.transaction() as connection: + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == name, _STATES.c.binding_generation == generation) + .values(phase=target, plugin_spec=None if target == "unloaded" else definition, last_error=None) + ) + if supervisor is not None: + supervisor.wake(name) + except Exception as error: + # No plugin configuration, credentials or exception input values in management status. + async with self.database.transaction() as connection: + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == name, _STATES.c.binding_generation == generation) + .values(phase="failed", last_error=type(error).__name__) + ) + + async def _maintain_search(self): + states = ARTIFACT_SEARCH_STATES_TABLE + while True: + try: + # Startup and initial registration discover related pairs needing initialization. + for family in tuple(self._index_refresh_families): + self._index_refresh_families.discard(family) + try: + complete = await self._refresh_search(family) + except SQLAlchemyError: + self._index_refresh_families.add(family) + raise + if not complete: + self._index_refresh_families.add(family) + async with self.database.transaction(consistent_snapshot=True) as connection: + pairs = ( + await connection.execute( + select(states.c.scope_id, states.c.family).where(states.c.state != "ready") + ) + ).all() + for scope, family in pairs: + async with self._index_locks.setdefault(family, asyncio.Lock()): + complete = False + try: + await self.index_builder.refresh(scope, family) + complete = await self.index_builder.build(scope, family) + except Exception: + self._index_failure(scope, family) + if complete: + self._index_failures.discard((scope, family)) + except SQLAlchemyError: + log_safely(logger, logging.WARNING, "Artifact search maintenance unavailable; retrying") + await asyncio.sleep(2) + + def _index_failure(self, scope, family): + if (scope, family) not in self._index_failures: + self._index_failures.add((scope, family)) + log_safely( + logger, logging.WARNING, "Artifact search rebuild incomplete; retrying while search is unavailable" + ) + + async def _refresh_search(self, family): + """Initialize missing targets on startup or first registration, without encoding.""" + async with self._index_locks.setdefault(family, asyncio.Lock()): + async with self.database.transaction(consistent_snapshot=True) as connection: + scopes = ( + ( + await connection.execute( + union( + select(ARTIFACT_HEADS_TABLE.c.scope_id).where(ARTIFACT_HEADS_TABLE.c.family == family), + select(ARTIFACT_SEARCH_STATES_TABLE.c.scope_id).where( + ARTIFACT_SEARCH_STATES_TABLE.c.family == family + ), + ) + ) + ) + .scalars() + .all() + ) + complete = True + for scope in scopes: + try: + await self.index_builder.refresh(scope, family) + except Exception: + complete = False + self._index_failure(scope, family) + return complete diff --git a/src/powercontext/server/authz/profiles.py b/src/powercontext/server/authz/profiles.py index c919f14347..c8d5c41e26 100644 --- a/src/powercontext/server/authz/profiles.py +++ b/src/powercontext/server/authz/profiles.py @@ -140,6 +140,23 @@ def subject_compatibility(self) -> dict[AccessRole, frozenset[str]]: } +def register_custom_artifact_access_family(family: str) -> None: + """Use framework-owned read/write semantics for a persisted custom family.""" + + if family not in ARTIFACT_FAMILY_PROFILES: + ARTIFACT_FAMILY_PROFILES[family] = ArtifactFamilyAccessProfile( + family=family, + enabled=True, + share_unit="artifact", + shareable_states=frozenset(), + base_action=AccessAction.ARTIFACT_READ, + additional_actions=frozenset(), + grantable_roles=frozenset(), + selector="forbidden", + mutation_semantics=frozenset({AccessAction.ARTIFACT_WRITE}), + ) + + def artifact_family_profile(resource: ResourceRef) -> ArtifactFamilyAccessProfile: """Validate an exact Artifact resource and return its enabled profile.""" diff --git a/src/powercontext/server/factory.py b/src/powercontext/server/factory.py index af56ad2ee7..49df0b1a2a 100644 --- a/src/powercontext/server/factory.py +++ b/src/powercontext/server/factory.py @@ -36,6 +36,7 @@ from powercontext.builtin.artifacts.skill import ExternalSkillProvider, SkillGenerator from powercontext.builtin.dream.generation import DreamGenerator from powercontext.builtin.inference import EmbeddingModel +from powercontext.builtin.persistence.records import RelationalRecordService from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.records import BaseOperationNotSupportedError from powercontext.builtin.runtime import ( @@ -61,6 +62,7 @@ from powercontext.paths import default_scheduler_path from powercontext.server.access import HttpAccessLogMiddleware from powercontext.server.app import _bind_evidence_access, access_readiness_checks, create_app +from powercontext.server.artifact_plugins import ArtifactPluginManager from powercontext.server.authentication import ( AuthenticationProvider, StaticBearerAuthenticationProvider, @@ -260,6 +262,36 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: await identity_repository.initialize() server_id = await identity_repository.load_or_create() _bind_evidence_access(runtime, active_access_control, resolved.access.mode) + if ( + config.runtime.artifact_processing_supervisor_mode == "global" + and config.runtime.artifact_processing_role != "api" + ): + records = runtime._records() + if not isinstance(records, RelationalRecordService): + raise TypeError("Artifact plugins require relational record storage") # noqa: TRY003 + + def bind_plugin_supervisor(supervisors) -> None: + runtime.artifact_processing_supervisor = supervisors + + app.state.artifact_plugin_manager = await resources.enter_async_context( + ArtifactPluginManager( + records.database, + config.database, + runtime.artifact_processing_supervisor, + lambda: build_worker_security( + resolved, + active_access_control, + legacy_static_principal=static_principal if legacy_static_admin else None, + enabled=True, + injected=configured_access_control is not None, + ), + on_supervisor=bind_plugin_supervisor, + plugins_dir=resolved.artifact_plugins_dir, + embedding_model=runtime._topic_memory_embedding_model, + inference_config=config.inference, + runtime_config=config.runtime, + ) + ) if active_access_control is not None: migrated, unresolved = await runtime._records().migrate_handoff_receipts( active_access_control.committed_receipt_identity, @@ -298,6 +330,7 @@ def current_capabilities() -> Capabilities: finally: _log_lifecycle("server.stopping", "PowerContext Server is stopping") readiness_probe.unbind() + app.state.artifact_plugin_manager = None app.state.application = None app.state.dashboard_runtime = None app.state.capability_provider = None diff --git a/src/powercontext/server/settings.py b/src/powercontext/server/settings.py index 962647be5b..5b573967eb 100644 --- a/src/powercontext/server/settings.py +++ b/src/powercontext/server/settings.py @@ -214,6 +214,7 @@ class ServerSettings(BaseSettings): handoff_generation_verification_secrets: tuple[SecretStr, ...] = Field(default=(), max_length=8, repr=False) code: CodeConfig = Field(default_factory=CodeConfig) runtime: RuntimeConfig = Field(default_factory=RuntimeConfig) + artifact_plugins_dir: Path | None = None database: DatabaseConfig = Field(default_factory=_default_database, discriminator="kind") handoff_report: HandoffReportConfig = Field(default_factory=HandoffReportConfig) inference: InferenceConfig = Field(default_factory=InferenceConfig) diff --git a/tests/builtin/persistence/test_artifact_indexes.py b/tests/builtin/persistence/test_artifact_indexes.py new file mode 100644 index 0000000000..f3856d4b45 --- /dev/null +++ b/tests/builtin/persistence/test_artifact_indexes.py @@ -0,0 +1,722 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Actual SQLite search, exact evidence and atomic projection publication.""" + +import asyncio +from contextlib import asynccontextmanager + +import pytest +from sqlalchemy import insert, select, update + +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, FragmentProjection, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchUnavailable, ArtifactSearchUnsupported +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.inference.models import EmbeddingResult +from powercontext.builtin.persistence.artifact_governance import ArtifactGovernanceRepository, ArtifactLifecycleState +from powercontext.builtin.persistence.artifact_indexes import ( + SCHEMAS, + STATES, + ArtifactIndexBuilder, + ArtifactProjectionPublisher, +) +from powercontext.builtin.persistence.artifacts import RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService +from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher + +PROFILE = EmbeddingProfile(profile_id="features-v1", model="fixture", dimension=2, normalization="none") + + +def declaration(*, version="1", vector_field="vector", profile=PROFILE): + return ArtifactSearchDeclaration( + version=version, + indexes=( + TextIndex(name="lexical", fields=("text",), projection_version="1"), + VectorIndex( + name="visual", + projection_version=version, + source="content", + profile=profile, + vector_field=vector_field, + profile_field="profile", + fragments=FragmentProjection(array_field="fragments", id_field="id"), + ), + ), + modes=("text", "vector", "hybrid"), + default_mode="text", + ) + + +@asynccontextmanager +async def store(config=None, model=None): + async with SQLiteProfile.open( + config or SQLiteConfig(), tables=BUILTIN_TABLES, load_vector_extension=True + ) as storage: + yield ( + storage.database, + ArtifactIndexBuilder(storage.database, model), + ArtifactProjectionPublisher(storage.database, model), + ) + + +async def register(db, family, search): + durable = {"family": family, "schema_version": "1", "search_fields": [], "search": search.model_dump(mode="json")} + async with db.transaction() as connection: + existing = await connection.scalar(select(SCHEMAS.c.family).where(SCHEMAS.c.family == family)) + if existing is None: + await connection.execute(insert(SCHEMAS).values(family=family, schema_version="1", declaration=durable)) + else: + await connection.execute(update(SCHEMAS).where(SCHEMAS.c.family == family).values(declaration=durable)) + + +async def write(db, publisher, scope, family, identity, content): + prepared = await publisher.prepare(scope, family, "1", content) + repository = custom_artifact_repository(family, prepared={identity: prepared}) + async with db.transaction() as connection: + draft = RepositoryArtifactDraft( + family=family, content=StoredArtifactContent(schema_version="1", content=content) + ) + try: + previous = await repository.latest(connection, scope, family, identity) + except RepositoryNotFoundError: + return await repository.create(connection, scope, identity, draft) + return await repository.revise(connection, scope, previous, draft) + + +def body(text, *vectors, profile=PROFILE): + return { + "text": text, + "fragments": [ + {"id": f"f{i}", "vector": vector, "profile": profile.model_dump(mode="json")} + for i, vector in enumerate(vectors) + ], + } + + +def service(db, family, model=None): + result = ArtifactSearchService(known_families=(family,)) + result.register(CustomArtifactSearcher(db, family, model)) + return result + + +def test_exact_vectors_collapse_fragments_before_limit_and_preserve_scope_history(): + async def run(): + async with store() as (db, builder, publisher): + for family in ("reports", "other"): + await register(db, family, declaration()) + await write(db, publisher, "scope", "reports", "a", body("数据库 timeout", *([[0, 0]] * 8))) + await write(db, publisher, "scope", "reports", "b", body("数据库 normal", [1, 0])) + await write(db, publisher, "foreign", "reports", "a", body("数据库 hidden", [0, 0])) + await write(db, publisher, "scope", "other", "a", body("数据库 hidden", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + request = { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]}, + "limit": 2, + "include_scores": True, + } + result = await search.search("scope", "reports", request) + assert [artifact.artifact_id for artifact in result.artifacts] == ["a", "b"] + assert result.matches[0].channel_scores["vector"].raw == 0 + old = result.artifacts[0].as_ref() + hybrid = await search.search("scope", "reports", {**request, "mode": "hybrid", "query": "数据库"}) + assert {artifact.artifact_id for artifact in hybrid.artifacts} == {"a", "b"} + assert all(0 <= hit.retrieval_score <= 1 for hit in hybrid.matches) + await write(db, publisher, "scope", "reports", "a", body("recovered")) + current = await search.search("scope", "reports", request) + assert [artifact.artifact_id for artifact in current.artifacts] == ["b"] + async with db.transaction() as connection: + historical = await custom_artifact_repository("reports").get(connection, "scope", old) + assert len(historical.content.content["fragments"]) == 8 + assert not (await search.search("scope", "reports", {"query": "timeout"})).matches + + asyncio.run(run()) + + +def test_initial_build_catches_insert_before_scan_cursor_and_publishes_atomically(): + async def run(): + async with store() as (db, builder, publisher): + + class ConcurrentEncoder: + profile = PROFILE + inserted = False + + async def embed(self, texts): + if not self.inserted: + self.inserted = True + # The text generation has finished its scan. This new identity is + # before its cursor and must be picked up by invalidation replay. + await write(db, publisher, "scope", "reports", "a", {"text": "original", "title": "new-index"}) + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + builder = ArtifactIndexBuilder(db, ConcurrentEncoder()) + initial = ArtifactSearchDeclaration( + version="1", + indexes=( + TextIndex(name="lexical", fields=("title",), projection_version="1"), + VectorIndex( + name="semantic", projection_version="1", source="text", profile=PROFILE, fields=("title",) + ), + ), + modes=("text",), + default_mode="text", + ) + await register(db, "reports", initial) + await write(db, publisher, "scope", "reports", "z", {"text": "original", "title": "new-index"}) + await builder.refresh("scope", "reports") + search = service(db, "reports") + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "original"}) + async with db.transaction() as connection: + await connection.exec_driver_sql( + "CREATE TRIGGER fail_release BEFORE UPDATE OF state ON pc_artifact_search_states WHEN new.state='ready' BEGIN SELECT RAISE(ABORT, 'release failure'); END" + ) + with pytest.raises(Exception, match="release failure"): + await builder.build("scope", "reports") + async with db.transaction() as connection: + assert ( + await connection.scalar( + select(STATES.c.state).where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + ) + == "building" + ) + await connection.exec_driver_sql("DROP TRIGGER fail_release") + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "original"}) + assert await builder.build("scope", "reports") + assert not (await search.search("scope", "reports", {"query": "original"})).matches + assert len((await search.search("scope", "reports", {"query": "new-index"})).matches) == 2 + + asyncio.run(run()) + + +def test_text_vectors_use_declared_encoder_and_unready_profile_never_falls_back(): + class Encoder: + profile = PROFILE + + async def embed(self, texts): + return EmbeddingResult(vectors=tuple((float(len(text)), 1.0) for text in texts)) + + async def run(): + model = Encoder() + async with store(model=model) as (db, builder, publisher): + search_config = ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", projection_version="1", source="text", profile=PROFILE, fields=("text",) + ), + ), + modes=("vector",), + default_mode="vector", + ) + await register(db, "reports", search_config) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库"}) + search = service(db, "reports", model) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "数据库"}) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + assert (await search.search("scope", "reports", {"query": "数据库"})).artifacts[0].artifact_id == "a" + with pytest.raises(ArtifactSearchUnavailable): + await service(db, "reports").search("scope", "reports", {"query": "数据库"}) + assert ( + await service(db, "reports").search( + "scope", "reports", {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [3, 1]}} + ) + ).matches + + asyncio.run(run()) + + +def test_query_errors_are_unsupported_and_declared_indexes_are_unavailable_during_repair(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + await write(db, publisher, "scope", "reports", "a", body("数据库", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + vector = {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]} + invalid = ( + {"query": "数据库", "mode": "text", "indexes": {"vector": ["visual"]}}, + {"query": "数据库", "indexes": {"text": []}}, + { + "query": "数据库", + "mode": "hybrid", + "query_vector": vector, + "fusion": {"method": "rrf", "params": {"weights": {"text": 0, "vector": 0}}}, + }, + {"mode": "vector", "query_vector": {**vector, "vector": [0]}}, + {"mode": "vector", "query": "数据库", "query_vector": vector}, + ) + for request in invalid: + with pytest.raises(ArtifactSearchUnsupported): + await search.search("scope", "reports", request) + await builder.refresh("scope", "reports", rebuild=True) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "数据库", "indexes": {"text": ["lexical"]}}) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "数据库"}) + + asyncio.run(run()) + + +def test_governance_removal_restore_and_rebuild_preserve_exact_history(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + original = await write(db, publisher, "scope", "reports", "a", body("数据库", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + async with db.transaction() as connection: + state = await ArtifactGovernanceRepository().transition( + connection, "scope", "reports", "a", 0, ArtifactLifecycleState.DEPRECATED, None + ) + assert not (await search.search("scope", "reports", {"query": "数据库"})).matches + # Restore requires all serving projections to be prepared outside its transaction. + with pytest.raises(ValueError, match="prepared before"): + async with db.transaction() as connection: + await ArtifactGovernanceRepository().transition( + connection, + "scope", + "reports", + "a", + state.governance_generation, + ArtifactLifecycleState.ACTIVE, + None, + ) + prepared = await publisher.prepare("scope", "reports", "1", original.content.content) + async with db.transaction() as connection: + governance = ArtifactGovernanceRepository(prepared_indexes={("scope", "reports", "a"): prepared}) + state = await governance.transition( + connection, + "scope", + "reports", + "a", + state.governance_generation, + ArtifactLifecycleState.ACTIVE, + None, + ) + assert (await search.search("scope", "reports", {"query": "数据库"})).artifacts[0].revision == 1 + await builder.refresh("scope", "reports", rebuild=True) + async with db.transaction() as connection: + await ArtifactGovernanceRepository().transition( + connection, + "scope", + "reports", + "a", + state.governance_generation, + ArtifactLifecycleState.RETIRED, + None, + ) + assert await builder.build("scope", "reports") + assert not (await search.search("scope", "reports", {"query": "数据库"})).matches + async with db.transaction() as connection: + assert ( + await custom_artifact_repository("reports").get(connection, "scope", original.as_ref()) + ).content == original.content + + asyncio.run(run()) + + +def test_incompatible_content_vector_write_preserves_search_and_history(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + first = await write(db, publisher, "scope", "reports", "a", body("数据库", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + next_profile = PROFILE.model_copy(update={"profile_id": "features-v2", "model": "second"}) + with pytest.raises(ValueError, match="profile differs"): + await write(db, publisher, "scope", "reports", "a", body("数据库", [0, 1], profile=next_profile)) + search = service(db, "reports") + request = { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + } + assert (await search.search("scope", "reports", request)).artifacts[0].revision == 1 + async with db.transaction() as connection: + assert ( + await custom_artifact_repository("reports").get(connection, "scope", first.as_ref()) + ).content == first.content + + asyncio.run(run()) + + +def test_vector_profile_is_validated_against_requested_index(): + async def run(): + async with store() as (db, builder, _): + alternate = PROFILE.model_copy(update={"profile_id": "alternate"}) + initial = declaration() + declared = ArtifactSearchDeclaration( + version="1", + indexes=( + *initial.indexes, + initial.indexes[1].model_copy(update={"name": "alternate", "profile": alternate}), + ), + modes=initial.modes, + default_mode="text", + defaults={"vector": {"vector": ("visual",)}, "hybrid": {"text": ("lexical",), "vector": ("visual",)}}, + ) + await register(db, "reports", declared) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + for selected, profile in (("visual", PROFILE), ("alternate", alternate)): + request = { + "mode": "vector", + "indexes": {"vector": [selected]}, + "query_vector": {"profile": profile.model_dump(mode="json"), "vector": [0, 0]}, + } + assert not (await search.search("scope", "reports", request)).matches + other_profile = alternate if selected == "visual" else PROFILE + request["query_vector"]["profile"] = other_profile.model_dump(mode="json") + with pytest.raises(ArtifactSearchUnsupported): + await search.search("scope", "reports", request) + + asyncio.run(run()) + + +def test_named_indexes_share_build_and_space_without_cross_matching(): + async def run(): + async with store() as (db, builder, publisher): + search_config = ArtifactSearchDeclaration( + version="1", + indexes=( + TextIndex(name="title", fields=("title",), projection_version="1"), + TextIndex(name="body", fields=("text",), projection_version="1"), + VectorIndex( + name="near", + source="content", + vector_field="v1", + profile_field="profile", + profile=PROFILE, + projection_version="1", + ), + VectorIndex( + name="far", + source="content", + vector_field="v2", + profile_field="profile", + profile=PROFILE, + projection_version="1", + ), + ), + modes=("text", "vector"), + default_mode="text", + defaults={"text": {"text": ("title",)}, "vector": {"vector": ("near",)}}, + ) + await register(db, "reports", search_config) + await write( + db, + publisher, + "scope", + "reports", + "a", + { + "title": "标题", + "text": "正文", + "v1": [0, 0], + "v2": [10, 0], + "profile": PROFILE.model_dump(mode="json"), + }, + ) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + for name, query, expected in (("title", "正文", False), ("body", "正文", True)): + assert ( + bool( + (await search.search("scope", "reports", {"query": query, "indexes": {"text": [name]}})).matches + ) + is expected + ) + for name, distance in (("near", 0), ("far", 10)): + found = await search.search( + "scope", + "reports", + { + "mode": "vector", + "indexes": {"vector": [name]}, + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]}, + "include_scores": True, + }, + ) + assert found.matches[0].channel_scores["vector"].raw == distance + + asyncio.run(run()) + + +def test_inflight_content_preparation_obeys_current_build_state(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + original = await write(db, publisher, "scope", "reports", "a", body("before", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + content = body("recovered", [1, 0]) + prepared = await publisher.prepare("scope", "reports", "1", content) + await builder.refresh("scope", "reports", rebuild=True) + # Preparation started when ready; commit now uses only the change log. + repository = custom_artifact_repository("reports", prepared={"a": prepared}) + draft = RepositoryArtifactDraft( + family="reports", content=StoredArtifactContent(schema_version="1", content=content) + ) + async with db.transaction() as connection: + revised = await repository.revise( + connection, "scope", await repository.get(connection, "scope", original.as_ref()), draft + ) + pending = await publisher.prepare("scope", "reports", "1", body("too-late", [2, 0])) + assert await builder.build("scope", "reports") + repository = custom_artifact_repository("reports", prepared={"a": pending}) + with pytest.raises(ValueError, match="prepare again"): + async with db.transaction() as connection: + await repository.revise( + connection, + "scope", + revised, + RepositoryArtifactDraft( + family="reports", + content=StoredArtifactContent(schema_version="1", content=body("too-late", [2, 0])), + ), + ) + assert (await service(db, "reports").search("scope", "reports", {"query": "recovered"})).artifacts[ + 0 + ].revision == 2 + + asyncio.run(run()) + + +def test_superseded_build_cannot_write_clean_or_publish_and_cleanup_resumes(): + async def run(): + from powercontext.builtin.persistence.artifact_indexes import COVERAGE, TEXTS, entries + + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + for scope, family in (("scope", "reports"), ("other-scope", "reports"), ("scope", "other-family")): + if family != "reports": + await register(db, family, declaration()) + await write(db, publisher, scope, family, "a", body("保留", [1, 0])) + await builder.refresh(scope, family) + assert await builder.build(scope, family) + await builder.refresh("scope", "reports", rebuild=True) + async with db.transaction() as connection: + old = ( + ( + await connection.execute( + select(STATES).where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + ) + ) + .mappings() + .one() + ) + assert not await builder.clean("scope", "reports", old["build_id"], limit=1) + # Restart repair with only part of the old rows removed. + await builder.refresh("scope", "reports", rebuild=True) + assert not await builder.clean("scope", "reports", old["build_id"]) + definition = entries(old["manifest"])[0][1]["index_def_id"] + assert not await builder._project_current("scope", "reports", old["build_id"], definition, "a") + assert not await builder._validate_and_publish("scope", "reports", old["build_id"]) + restarted = ArtifactIndexBuilder(db) + assert await restarted.build("scope", "reports") + for scope, family in (("scope", "reports"), ("other-scope", "reports"), ("scope", "other-family")): + assert (await service(db, family).search(scope, family, {"query": "保留"})).matches + async with db.transaction() as connection: + current = ( + ( + await connection.execute( + select(STATES).where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + ) + ) + .mappings() + .one() + ) + assert current["cleanup_targets"] == [] + for table in (TEXTS, COVERAGE): + assert not ( + await connection.execute( + select(table).where( + table.c.scope_id == "scope", + table.c.family == "reports", + table.c.build_id != current["build_id"], + ) + ) + ).all() + # A crashed final validator must not leave the build permanently occupied. + await restarted.refresh("scope", "reports", rebuild=True) + async with db.transaction() as connection: + await connection.execute( + update(STATES) + .where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + .values(validation_token="interrupted") # noqa: S106 - Persisted build fencing token, not a credential. + ) + assert await ArtifactIndexBuilder(db).build("scope", "reports") + + asyncio.run(run()) + + +def test_repair_restart_rejects_an_inflight_encoder_and_keeps_new_build_unavailable(): + async def run(): + entered, release = asyncio.Event(), asyncio.Event() + + class Encoder: + profile = PROFILE + + async def embed(self, texts): + entered.set() + await release.wait() + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + async with store(model=Encoder()) as (db, builder, publisher): + config = ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", fields=("text",), source="text", profile=PROFILE, projection_version="1" + ), + ), + modes=("vector",), + default_mode="vector", + ) + await register(db, "reports", config) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库"}) + await builder.refresh("scope", "reports") + old_task = asyncio.create_task(builder.build("scope", "reports")) + try: + await asyncio.wait_for(entered.wait(), 5) + await builder.refresh("scope", "reports", rebuild=True) + release.set() + assert not await old_task + request = {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}} + with pytest.raises(ArtifactSearchUnavailable): + await service(db, "reports").search("scope", "reports", request) + assert await builder.build("scope", "reports") + assert (await service(db, "reports").search("scope", "reports", request)).matches + finally: + release.set() + await old_task + + asyncio.run(run()) + + +@pytest.mark.parametrize("during_build", [False, True]) +def test_query_policy_update_preserves_build_and_does_not_reencode(during_build): + async def run(): + entered, release = asyncio.Event(), asyncio.Event() + + class Encoder: + profile = PROFILE + encoded = False + + async def embed(self, texts): + assert not self.encoded, "query policy must not re-encode stored content" + self.encoded = True + entered.set() + await release.wait() + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + async with store(model=Encoder()) as (db, builder, publisher): + initial = ArtifactSearchDeclaration( + version="1", + indexes=( + TextIndex(name="title", fields=("title",), projection_version="1"), + TextIndex(name="body", fields=("text",), projection_version="1"), + VectorIndex( + name="semantic", fields=("text",), source="text", profile=PROFILE, projection_version="1" + ), + ), + modes=("text", "vector"), + default_mode="text", + defaults={"text": {"text": ("title",)}, "vector": {"vector": ("semantic",)}}, + ) + await register(db, "reports", initial) + await write(db, publisher, "scope", "reports", "a", {"title": "标题", "text": "正文"}) + await builder.refresh("scope", "reports") + build = asyncio.create_task(builder.build("scope", "reports")) + try: + async with asyncio.timeout(5): + await entered.wait() + if not during_build: + release.set() + assert await build + async with db.transaction() as connection: + before = dict((await connection.execute(select(STATES))).mappings().one()) + changed = initial.model_copy( + update={ + "default_mode": "vector", + "defaults": {"text": {"text": ("body",)}, "vector": {"vector": ("semantic",)}}, + "candidate_limit": 17, + "rrf_k": 11, + "text_weight": 2, + "vector_weight": 3, + } + ) + await register(db, "reports", changed) + await builder.refresh("scope", "reports") + async with db.transaction() as connection: + after = dict((await connection.execute(select(STATES))).mappings().one()) + assert after == before # The build identity, progress and readiness must survive policy changes. + release.set() + assert await build + assert await builder.build("scope", "reports") + search = service(db, "reports") + result = await search.search( + "scope", + "reports", + { + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + }, + ) + assert [item.artifact_id for item in result.artifacts] == ["a"] + assert (await search.search("scope", "reports", {"mode": "text", "query": "正文"})).artifacts + assert not (await search.search("scope", "reports", {"mode": "text", "query": "标题"})).artifacts + finally: + release.set() + await build + + asyncio.run(run()) + + +def test_inflight_query_keeps_policy_while_next_query_uses_update(): + async def run(): + entered, release = asyncio.Event(), asyncio.Event() + + class QueryEncoder: + profile = PROFILE + + async def embed(self, texts): + entered.set() + await release.wait() + return EmbeddingResult(vectors=((1.0, 0.0),)) + + async with store() as (db, builder, publisher): + initial = declaration().model_copy(update={"default_mode": "vector"}) + await register(db, "reports", initial) + await write(db, publisher, "scope", "reports", "a", body("数据库", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports", QueryEncoder()) + request = {"query": "no lexical match"} + pending = asyncio.create_task(search.search("scope", "reports", request)) + try: + async with asyncio.timeout(5): + await entered.wait() + await register(db, "reports", initial.model_copy(update={"modes": ("text",), "default_mode": "text"})) + release.set() + assert [item.artifact_id for item in (await pending).artifacts] == ["a"] + assert not (await search.search("scope", "reports", request)).artifacts + finally: + release.set() + await pending + + asyncio.run(run()) diff --git a/tests/builtin/persistence/test_custom_artifact_search.py b/tests/builtin/persistence/test_custom_artifact_search.py new file mode 100644 index 0000000000..1b494495e5 --- /dev/null +++ b/tests/builtin/persistence/test_custom_artifact_search.py @@ -0,0 +1,112 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Initialize missing search tables in existing Artifact storage, preserving exact historical content.""" + +import asyncio +import sqlite3 + +from sqlalchemy import insert + +from powercontext.artifacts import Artifact +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.artifacts import ArtifactRepository, RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_SEARCH_TABLES, + BUILTIN_TABLES, +) +from tests.builtin.persistence.test_artifact_indexes import service + + +def test_existing_database_adds_search_tables_and_rebuilds_legacy_declarations(tmp_path): + path = tmp_path / "existing.db" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{path}") + + async def legacy(): + tables = tuple(table for table in BUILTIN_TABLES if table not in ARTIFACT_SEARCH_TABLES) + async with SQLiteProfile.open(config, tables=tables) as profile: + artifact_type = type("Legacy", (Artifact[StoredArtifactContent],), {"family": "reports"}) + repository = ArtifactRepository((artifact_type,)) + async with profile.database.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family="reports", + schema_version="1", + declaration={"family": "reports", "schema_version": "1", "search_fields": ["body"]}, + ) + ) + first = await repository.create( + connection, + "scope", + "a", + RepositoryArtifactDraft( + family="reports", + content=StoredArtifactContent(schema_version="1", content={"body": "数据库 TIMEOUT"}), + ), + ) + await repository.revise( + connection, + "scope", + first, + RepositoryArtifactDraft( + family="reports", + content=StoredArtifactContent(schema_version="1", content={"body": "连接恢复 RECOVERED"}), + ), + ) + + asyncio.run(legacy()) + with sqlite3.connect(path) as connection: + before = connection.execute("SELECT * FROM pc_artifacts ORDER BY revision").fetchall() + assert not connection.execute("SELECT 1 FROM sqlite_master WHERE name='pc_artifact_search_states'").fetchall() + + async def rebuilt(): + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as profile: + builder = ArtifactIndexBuilder(profile.database) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(profile.database, "reports") + assert not (await search.search("scope", "reports", {"query": "TIMEOUT"})).matches + result = await search.search("scope", "reports", {"query": "RECOVERED"}) + assert result.artifacts[0].revision == 2 + assert result.artifacts[0].content.model_dump() == {"body": "连接恢复 RECOVERED"} + + asyncio.run(rebuilt()) + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT * FROM pc_artifacts ORDER BY revision").fetchall() == before + assert not connection.execute("SELECT 1 FROM sqlite_master WHERE name='pc_artifact_search_schema'").fetchall() + connection.execute("DROP TRIGGER pc_artifact_search_fts_update") + # Missing full-text support is recreated; repeated initialization preserves exact content. + asyncio.run(rebuilt()) + + +def test_intermediate_search_schema_is_rejected_without_deleting_existing_rows(tmp_path): + import pytest + + path = tmp_path / "incompatible.db" + + async def initialize_database(): + tables = tuple(table for table in BUILTIN_TABLES if table not in ARTIFACT_SEARCH_TABLES) + async with SQLiteProfile.open(SQLiteConfig(url=f"sqlite+aiosqlite:///{path}"), tables=tables): + pass + + asyncio.run(initialize_database()) + with sqlite3.connect(path) as connection: + connection.execute( + "CREATE TABLE pc_artifact_search_states (scope_id TEXT, family TEXT, release_id INTEGER, serving_manifest TEXT, desired_manifest TEXT, change_seq INTEGER, PRIMARY KEY(scope_id, family))" + ) + connection.execute("INSERT INTO pc_artifact_search_states VALUES ('scope', 'reports', 3, '{}', '{}', 9)") + before = connection.execute("SELECT * FROM pc_artifact_search_states").fetchall() + + async def open_old_database(): + with pytest.raises(ValueError, match=r"incompatible.*migration"): + async with SQLiteProfile.open(SQLiteConfig(url=f"sqlite+aiosqlite:///{path}"), tables=BUILTIN_TABLES): + pytest.fail("Old search layout must not be treated as upgraded") + + asyncio.run(open_old_database()) + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT * FROM pc_artifact_search_states").fetchall() == before diff --git a/tests/builtin/persistence/test_mysql_schema.py b/tests/builtin/persistence/test_mysql_schema.py index 5c0c5e7a79..27332865c7 100644 --- a/tests/builtin/persistence/test_mysql_schema.py +++ b/tests/builtin/persistence/test_mysql_schema.py @@ -165,7 +165,9 @@ def test_documented_obloader_restore_layers_are_parent_first() -> None: assert len(set(restore_plans)) == 1 restore_layers = restore_plans[0] - _assert_restore_layers_are_parent_first(restore_layers, BUILTIN_TABLES) + _assert_restore_layers_are_parent_first( + restore_layers, tuple(table for table in BUILTIN_TABLES if table.name != "pc_artifact_vectors") + ) def test_every_mysql_utf8mb4_key_stays_below_the_innodb_limit() -> None: diff --git a/tests/builtin/runtime/test_artifact_read_context.py b/tests/builtin/runtime/test_artifact_read_context.py new file mode 100644 index 0000000000..5f69441cc4 --- /dev/null +++ b/tests/builtin/runtime/test_artifact_read_context.py @@ -0,0 +1,267 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""The shared read context preserves scope, family and caller permissions.""" + +import asyncio +from contextlib import AsyncExitStack +from datetime import UTC, datetime +from typing import cast + +import pytest +from sqlalchemy import insert, select + +from powercontext.artifact_plugins import GenerationContext +from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.search import ArtifactSearchUnsupported +from powercontext.builtin.artifacts.profile.models import Profile, ProfileContent, ProfileGeneration +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.artifacts import ArtifactRepository, RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import ( + StoredArtifactContent, + custom_artifact_repository, +) +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.sqlite.experience_index import SQLiteExperienceFTSIndex +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE, BUILTIN_TABLES, SOURCE_CURSORS_TABLE +from powercontext.builtin.records import BaseValueNotFoundError, InvalidBaseAccessRequestError +from powercontext.builtin.runtime.artifact_reads import ArtifactReadSession +from powercontext.server.authz import ( + AccessAuditContext, + AccessControlService, + AccessDecision, + AccessDeniedError, + AccessRole, + BuiltinAuthorizationProvider, + CreateBinding, + PrincipalRef, + ResourceRef, +) +from powercontext.server.authz.composition import open_builtin_access_control +from powercontext.server.authz.profiles import register_custom_artifact_access_family +from powercontext.server.authz.repository import RelationalAccessRepository + + +class FilteredProvider(BuiltinAuthorizationProvider): + async def check_batch(self, requests, /): + decisions = await super().check_batch(requests) + return tuple( + AccessDecision(False, "restricted-artifact", decision.policy_revision) + if request.resource.artifact_id == "a-hidden" + else decision + for request, decision in zip(requests, decisions, strict=True) + ) + + +def test_generation_reads_share_permission_filtered_candidates_and_exact_evidence(tmp_path): + async def scenario(): + admin = PrincipalRef(type="service", id="admin") + alice, bob = PrincipalRef(type="user", id="alice"), PrincipalRef(type="user", id="bob") + audit = AccessAuditContext(transport="test", operation="artifact-reads") + family = "context-items" + register_custom_artifact_access_family(family) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'reads.db'}") + async with AsyncExitStack() as resources: + profile = await resources.enter_async_context(SQLiteProfile.open(config, tables=BUILTIN_TABLES)) + access = await resources.enter_async_context( + open_builtin_access_control(config, bootstrap_administrators=(admin,)) + ) + await access.create_binding( + admin, + CreateBinding( + subject=alice, + resource=ResourceRef.scope("scope"), + role=AccessRole.SCOPE_VIEWER, + idempotency_key="scope-viewer", + ), + context=audit, + ) + repository = custom_artifact_repository(family) + async with profile.database.transaction() as connection: + await SQLiteExperienceFTSIndex().initialize(connection) + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={"search_fields": ["text"], "schema_version": "1"}, + ) + ) + for scope, identity in (("scope", "a-hidden"), ("scope", "b-visible"), ("other", "foreign")): + await repository.create( + connection, + scope, + identity, + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent(schema_version="1", content={"text": "数据库 TIMEOUT"}), + ), + ) + builder = ArtifactIndexBuilder(profile.database) + await builder.refresh("scope", family) + assert await builder.build("scope", family) + for identity, owner in (("a-hidden", bob), ("b-visible", alice)): + await access.establish_artifact_owner( + ResourceRef.artifact("scope", family=family, artifact_id=identity), + owner, + idempotency_key=identity, + context=audit, + ) + access = AccessControlService( + FilteredProvider(cast(RelationalAccessRepository, access.relationships)), + relationships=access.relationships, + audit=access.audit, + ) + reads = ArtifactReadSession( + profile.database, "scope", (family,), access=access, principal=alice, audit=audit + ) + context = GenerationContext( + "scope", resources, reads.read, list_artifacts=reads.list, search_artifacts=reads.search + ) + page = await context.list_artifacts(family, limit=1) + assert [item.artifact_id for item in page.items] == ["b-visible"] + assert page.next_cursor is None + matches = await context.search_artifacts(family, "数据库", limit=1) + assert [hit.artifact_ref.artifact_id for hit in matches.matches] == ["b-visible"] + assert matches.artifacts is not None + assert matches.artifacts[0].content.model_dump() == {"text": "数据库 TIMEOUT"} + assert not reads.refs + snapshot = await context.read_revision(matches.matches[0].artifact_ref) + assert snapshot.content["text"] == "数据库 TIMEOUT" + assert reads.refs == {(family, "b-visible", 1)} + with pytest.raises(AccessDeniedError): + await context.read_latest(family, "a-hidden") + with pytest.raises(InvalidBaseAccessRequestError): + await context.read_latest("undeclared", "any") + with pytest.raises(InvalidBaseAccessRequestError): + await context.list_artifacts("undeclared") + with pytest.raises(InvalidBaseAccessRequestError): + await context.search_artifacts("undeclared", "数据库") + assert await context.read_latest(family, "foreign") is None + denied = ArtifactReadSession( + profile.database, "other", (family,), access=access, principal=alice, audit=audit + ) + foreign_context = GenerationContext( + "other", resources, denied.read, list_artifacts=denied.list, search_artifacts=denied.search + ) + with pytest.raises(AccessDeniedError): + await foreign_context.read_latest(family, "foreign") + with pytest.raises(AccessDeniedError): + await foreign_context.list_artifacts(family) + with pytest.raises(AccessDeniedError): + await foreign_context.search_artifacts(family, "数据库") + + asyncio.run(scenario()) + + +def test_generation_context_reads_fixed_history_across_families_and_pages_current_references(tmp_path): + async def scenario(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'history.db'}") + async with AsyncExitStack() as resources: + profile = await resources.enter_async_context(SQLiteProfile.open(config, tables=BUILTIN_TABLES)) + items = custom_artifact_repository("context-items") + roots = custom_artifact_repository("context-roots") + async with profile.database.transaction() as connection: + await SQLiteExperienceFTSIndex().initialize(connection) + for family in ("context-items", "context-roots"): + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={ + "schema_version": "1", + "search_fields": ["text"] if family == "context-items" else [], + }, + ) + ) + first = await items.create( + connection, + "scope", + "a", + RepositoryArtifactDraft( + family="context-items", + content=StoredArtifactContent(schema_version="1", content={"text": "旧证据"}), + ), + ) + await roots.create( + connection, + "scope", + "root", + RepositoryArtifactDraft( + family="context-roots", + content=StoredArtifactContent( + schema_version="1", content={"ref": first.as_ref().model_dump(mode="json")} + ), + ), + ) + for identity in ("b", "c"): + await items.create( + connection, + "scope", + identity, + RepositoryArtifactDraft( + family="context-items", + content=StoredArtifactContent(schema_version="1", content={"text": "其他"}), + ), + ) + builtin = await ArtifactRepository((Profile,)).create( + connection, + "scope", + "profile", + RepositoryArtifactDraft( + family="profile", + content=ProfileContent( + content="使用中文", + generation=ProfileGeneration(mode="manual_create", created_at=datetime.now(UTC)), + ), + ), + ) + await connection.execute( + insert(SOURCE_CURSORS_TABLE).values( + scope_id="scope", binding_name="reports", cursor=b'{"through":7}', generation=3 + ) + ) + reads = ArtifactReadSession(profile.database, "scope", ("context-items", "context-roots", "profile")) + context = GenerationContext( + "scope", resources, reads.read, list_artifacts=reads.list, search_artifacts=reads.search + ) + root = await context.read_latest("context-roots", "root") + assert root is not None + fixed_ref = ArtifactRef.model_validate(root.content["ref"]) + async with profile.database.transaction() as connection: + await items.revise( + connection, + "scope", + first, + RepositoryArtifactDraft( + family="context-items", + content=StoredArtifactContent(schema_version="1", content={"text": "数据库 NEW"}), + ), + ) + builder = ArtifactIndexBuilder(profile.database) + await builder.refresh("scope", "context-items") + assert await builder.build("scope", "context-items") + first_page = await context.list_artifacts("context-items", limit=2) + assert [(ref.artifact_id, ref.revision) for ref in first_page.items] == [("a", 2), ("b", 1)] + assert first_page.next_cursor is not None + last_page = await context.list_artifacts("context-items", limit=2, cursor=first_page.next_cursor) + assert [(ref.artifact_id, ref.revision) for ref in last_page.items] == [("c", 1)] + assert last_page.next_cursor is None + assert not (await context.search_artifacts("context-items", "旧证据")).matches + current = await context.search_artifacts("context-items", "数据库") + assert current.matches[0].artifact_ref.revision == 2 + assert (await context.read_revision(current.matches[0].artifact_ref)).content["text"] == "数据库 NEW" + assert (await context.read_revision(fixed_ref)).content["text"] == "旧证据" + snapshot = await context.read_revision(builtin.as_ref()) + assert snapshot.schema_version is None + assert snapshot.content == builtin.content.model_dump(mode="json", by_alias=True) + with pytest.raises(BaseValueNotFoundError): + await context.read_revision(ArtifactRef(family="context-items", artifact_id="a", revision=99)) + with pytest.raises(ArtifactSearchUnsupported): + await context.search_artifacts("context-items", "数据库", mode="hybrid") + async with profile.database.transaction() as connection: + cursor = (await connection.execute(select(SOURCE_CURSORS_TABLE))).mappings().one() + assert cursor["cursor"] == b'{"through":7}' and cursor["generation"] == 3 + + asyncio.run(scenario()) diff --git a/tests/e2e/test_artifact_index_backends.py b/tests/e2e/test_artifact_index_backends.py new file mode 100644 index 0000000000..5671056c63 --- /dev/null +++ b/tests/e2e/test_artifact_index_backends.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Real backend projection transactions, native initialization and exact search.""" + +import asyncio +import json + +import pytest + +from powercontext.artifacts.search import ArtifactSearchUnavailable +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder, ArtifactProjectionPublisher +from powercontext.builtin.persistence.oceanbase import OceanBaseProfile +from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher, CustomArtifactSearchRequest +from tests.builtin.persistence.test_artifact_indexes import PROFILE, body, declaration, register, service, write + + +def test_search_projection_backends_require_native_provisioning_and_preserve_revisions(migration_backend): + config = migration_backend.config + + def open_storage(): + if isinstance(config, SQLiteConfig): + return SQLiteProfile.open(config, tables=BUILTIN_TABLES, load_vector_extension=True) + if isinstance(config, SeekDBConfig): + return SeekDBProfile.open(config, tables=BUILTIN_TABLES) + return OceanBaseProfile.open(config, tables=BUILTIN_TABLES) + + async def declare(): + async with open_storage() as storage: + db = storage.database + await register(db, "reports", declaration()) + publisher = ArtifactProjectionPublisher(db) + await write(db, publisher, "scope", "reports", "a", body("数据库 timeout", *([[0, 0]] * 8))) + await write(db, publisher, "scope", "reports", "b", body("数据库 normal", [1, 0])) + builder = ArtifactIndexBuilder(db) + await builder.refresh("scope", "reports") + if isinstance(config, SQLiteConfig): + assert await builder.build("scope", "reports") + else: + with pytest.raises(ArtifactSearchUnavailable, match="database initialization"): + await builder.build("scope", "reports") + + asyncio.run(declare()) + + async def verify(): + async with open_storage() as storage: + db = storage.database + builder = ArtifactIndexBuilder(db) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + vector = {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]} + request = {"mode": "vector", "query_vector": vector, "limit": 2, "include_scores": True} + found = await search.search("scope", "reports", request) + assert [item.artifact_id for item in found.artifacts] == ["a", "b"] + assert found.matches[0].channel_scores["vector"].raw == 0 + hybrid = await search.search("scope", "reports", {**request, "mode": "hybrid", "query": "数据库"}) + assert {item.artifact_id for item in hybrid.artifacts} == {"a", "b"} + # Pause only to order the concurrent operations; all reads and writes use the real backend. + reading, published = asyncio.Event(), asyncio.Event() + + class SnapshotReader(CustomArtifactSearcher): + async def _candidates(self, *args, **kwargs): + reading.set() + await published.wait() + return await super()._candidates(*args, **kwargs) + + reader = asyncio.create_task( + SnapshotReader(db, "reports").search( + "scope", + CustomArtifactSearchRequest.model_validate_json( + json.dumps({**request, "mode": "hybrid", "query": "数据库"}) + ), + ) + ) + try: + await asyncio.wait_for(reading.wait(), 10) + await builder.refresh("scope", "reports", rebuild=True) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", request) + assert await builder.build("scope", "reports") + finally: + published.set() + snapshot = await reader + assert {item.artifact_id for item in snapshot.artifacts} == {"a", "b"} + publisher = ArtifactProjectionPublisher(db) + await write(db, publisher, "scope", "reports", "a", body("recovered")) + assert [item.artifact_id for item in (await search.search("scope", "reports", request)).artifacts] == ["b"] + assert not (await search.search("scope", "reports", {"query": "timeout"})).matches + + asyncio.run(verify()) diff --git a/tests/e2e/test_artifact_plugin_search.py b/tests/e2e/test_artifact_plugin_search.py new file mode 100644 index 0000000000..9de8ae5d50 --- /dev/null +++ b/tests/e2e/test_artifact_plugin_search.py @@ -0,0 +1,403 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Custom Artifact search through generation, revisions, unload and index recovery.""" + +import asyncio +import json +import shutil +from pathlib import Path + +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from tests.e2e.test_artifact_plugins import ( + EXAMPLE, + REGISTRATION, + _wait_for_search_ready, + change, + create_scope, + deploy_plugin, + enable_all, + plugin_state, + publish_plugin, + reload_plugin, + running_server, +) + +_PIPELINE = """ + +class SearchPipeline: + def __init__(self, ctx): + self.ctx = ctx + + async def generate(self, batch): + drafts = [] + for source in batch.sources: + value = json.loads(TextEvidence.model_validate(source.value).content) + artifact_id = value.pop("artifact_id") + previous = await self.ctx.read_latest("incident-report", artifact_id) + drafts.append(ArtifactDraft( + family="incident-report", schema_version=families[0].schema_version, + artifact_id=artifact_id, + expected_revision=0 if previous is None else previous.ref.revision, + content=value, source_refs=(source.ref,), + )) + return drafts + +async def create_pipeline(ctx, config): + return SearchPipeline(ctx) +""" + + +async def _write_report(client, scope, source_id, artifact_id, body, *, title="Report", revision=1, fragments=None): + response = await client.post( + "/v1/sources/content", + json={ + "scope_id": scope, + "source_id": source_id, + "content": json.dumps( + { + "artifact_id": artifact_id, + "title": title, + "evidence_text": body, + "private_text": "不可检索 secretword", + **({"fragments": fragments} if fragments is not None else {}), + }, + ensure_ascii=False, + ), + }, + ) + assert response.status_code == 202, response.text + async with asyncio.timeout(45): + while True: + page = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report") + assert page.status_code == 200, page.text + if any( + item["artifact_id"] == artifact_id and item["revision"] == revision for item in page.json()["items"] + ): + response = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{artifact_id}") + assert response.status_code == 200, response.text + return response.json() + await asyncio.sleep(0.2) + + +async def _search(client, scope, query): + response = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": query}) + assert response.status_code == 200, response.text + return response.json() + + +def test_custom_search_preserves_full_content_and_rebuilds_retained_revisions(tmp_path: Path) -> None: + asyncio.run(_search_through_lifecycle(tmp_path)) + + +async def _search_through_lifecycle(directory: Path) -> None: + async with running_server(directory) as client: + scope = await create_scope(client, "search") + other_scope = await create_scope(client, "other-search") + plugin = deploy_plugin(EXAMPLE, directory / "search-v1", [scope, other_scope]) + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text().replace( + "class IncidentReport(BaseModel):", "class IncidentReport(BaseModel):\n private_text: str" + ) + + _PIPELINE, + encoding="utf-8", + ) + await reload_plugin(client, directory, plugin, 0) + await enable_all(client) + state = await plugin_state(client, "enabled") + removed = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/query", json={"query": "数据库"}) + assert removed.status_code == 405 # The remaining GET /{artifact_id} route has no POST operation. + for invalid in ( + {"query": "数据库", "limit": 201}, + {"query": "数据库", "indexes": {"text": []}}, + {"query": "数据库", "mode": "fts"}, + ): + response = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/search", json=invalid) + assert response.status_code == 422, response.text + + original = "数据库连接超时 TIMEOUT，重试后恢复。" # noqa: RUF001 + first = await _write_report(client, scope, "first", "report", original, title="故障记录") + # The first write creates the durable build request; empty scopes no longer + # receive prebuilt releases when the plugin is loaded. + await _wait_for_search_ready(client, scope, "数据库") + for query in ("数据库", "连接超时", "timeout", "数据库 timeout", "故障记录"): + hits = (await _search(client, scope, query))["results"] + assert [{key: hit[key] for key in ("family", "artifact_id", "revision")} for hit in hits] == [ + {"family": "incident-report", "artifact_id": "report", "revision": 1} + ] + assert hits[0]["content"] == first["content"] + assert set(hits[0]) == {"family", "artifact_id", "revision", "content", "lineage"} + for query in ("不存在的英文token", "secretword", "不可检索", "!!!"): + assert (await _search(client, scope, query))["results"] == [] + assert (await _search(client, other_scope, "数据库"))["results"] == [] + + # Normalization can change code point counts; excerpts must still use the exact original. + for index, (needle, query) in enumerate(( + ("磁盘告警", "磁盘告警"), + ("Straße", "strasse"), + ("Cafe\u0301", "café"), + ("customer_id", "customer_id"), + ("pcu123", "pcu123"), + )): + body = "prefix ß e\u0301. " * 80 + needle + " suffix." * 80 + await _write_report(client, scope, f"long-{index}", f"long-{index}", body) + hits = (await _search(client, scope, query))["results"] + assert len(hits) == 1 + assert hits[0]["content"]["evidence_text"] == body + for query in ("cafe", "customer", "id"): + assert (await _search(client, scope, query))["results"] == [] + + # Replacing a head must remove its old words while retaining the exact historical revision. + await _write_report(client, scope, "updated", "report", "网络恢复正常 recovered", revision=2) + assert (await _search(client, scope, "数据库 timeout"))["results"] == [] + assert (await _search(client, scope, "网络恢复"))["results"][0]["revision"] == 2 + historical = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/report/revisions/1") + assert historical.status_code == 200 + assert historical.json()["content"] == first["content"] + + # A schema bump must not bypass the fixed search declaration. + replacement = deploy_plugin(plugin, directory / "search-v2", [scope, other_scope], version="2") + entry = replacement / "plugin.py" + entry.write_text( + entry + .read_text() + .replace('schema_version="1"', 'schema_version="2"') + .replace('fields=("title", "evidence_text")', 'fields=("evidence_text",)'), + encoding="utf-8", + ) + publish_plugin(directory, replacement) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await _search(client, scope, "网络恢复"))["results"][0]["revision"] == 2 + # Code-only reload remains supported with exactly the same search declaration. + entry.write_text((plugin / "plugin.py").read_text()) + state = await reload_plugin(client, directory, replacement, state["binding_generation"], "enabled") + await _write_report(client, scope, "after-reload", "after-reload", "缓存刷新 complete") + await _wait_for_search_ready(client, scope, "缓存刷新") + disabled = await change(client, "disable", state["binding_generation"], "disabled") + assert (await _search(client, scope, "缓存刷新"))["results"][0]["content"][ + "evidence_text" + ] == "缓存刷新 complete" + await change(client, "unload", disabled["binding_generation"], "unloaded") + before = { + query: await _search(client, scope, query) + for query in ("网络恢复", "磁盘告警", "strasse", "café", "customer_id", "pcu123", "缓存刷新") + } + saved = (await client.get(f"/v1/scopes/{scope}/artifacts/incident-report")).json() + + # Persist a rebuild request, then recover it without either plugin's code. + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{directory / 'server.db'}"), tables=BUILTIN_TABLES + ) as storage: + await ArtifactIndexBuilder(storage.database).refresh(scope, "incident-report", rebuild=True) + shutil.rmtree(plugin) + shutil.rmtree(replacement) + async with running_server(directory) as client: + assert (await client.get(REGISTRATION)).json()["phase"] == "unloaded" + assert (await client.get(f"/v1/scopes/{scope}/artifacts/incident-report")).json() == saved + # Server startup no longer waits for rebuilding retained data. Only the + # affected search can be unavailable until its complete manifest is published. + await _wait_for_search_ready(client, scope, "网络恢复") + for query, expected in before.items(): + assert await _search(client, scope, query) == expected + assert (await _search(client, scope, "数据库 timeout hiddenfield secretword"))["results"] == [] + assert (await client.get(f"/v1/scopes/{scope}/artifacts/incident-report")).json() == saved + + +def test_pending_vector_index_suspends_all_search_but_not_generation(tmp_path): + async def run(): + async with running_server(tmp_path) as client: + scope = await create_scope(client, "pending-search") + plugin = deploy_plugin(EXAMPLE, tmp_path / "initial", [scope]) + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text().replace( + "class IncidentReport(BaseModel):", "class IncidentReport(BaseModel):\n private_text: str" + ) + + _PIPELINE + ) + entry.write_text( + entry.read_text() + + """ +from dataclasses import replace +from powercontext.artifact_plugins import VectorIndex +from powercontext.builtin.artifacts.memory import EmbeddingProfile +families = (replace(families[0], search=ArtifactSearchDeclaration( + version="2", indexes=(*families[0].search.indexes, VectorIndex( + name="semantic", projection_version="1", source="text", fields=("evidence_text",), + profile=EmbeddingProfile(profile_id="unavailable", model="unconfigured", dimension=2), + )), modes=("text", "vector"), default_mode="text", +)),) +""" + ) + await reload_plugin(client, tmp_path, plugin, 0) + await enable_all(client) + await plugin_state(client, "enabled") + await _write_report(client, scope, "first", "report", "outage") + response = await client.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"mode": "vector", "query": "outage"} + ) + assert response.status_code == 503 + response = await client.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": "outage"} + ) + assert response.status_code == 503 + latest = await _write_report(client, scope, "second", "report", "recovered", revision=2) + assert latest["revision"] == 2 + historical = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/report/revisions/1") + assert historical.status_code == 200, historical.text + assert historical.json()["revision"] == 1 + assert historical.json()["content"]["evidence_text"] == "outage" + response = await client.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": "recovered"} + ) + assert response.status_code == 503 + + asyncio.run(run()) + + +def test_index_definitions_stay_fixed_while_query_policy_survives_reload_and_restart(tmp_path): + from tests.e2e.test_artifact_plugins import artifacts, capture + + def deploy(version, profile_id, index_name="visual"): + plugin = deploy_plugin(EXAMPLE, tmp_path / version, None, version=version) + entry = plugin / "plugin.py" + entry.write_text( + entry + .read_text() + .replace( + "class IncidentReport(BaseModel):", + "from powercontext.builtin.artifacts.memory import EmbeddingProfile\n" + f'VECTOR_PROFILE = EmbeddingProfile(profile_id="{profile_id}", model="fixture", dimension=2)\n' + "class IncidentReport(BaseModel):\n vector: list[float]\n profile: EmbeddingProfile", + ) + .replace('title=f"', 'vector=[1.0, 0.0], profile=VECTOR_PROFILE, title=f"') + + f''' +from dataclasses import replace +from powercontext.artifact_plugins import VectorIndex +families = (replace(families[0], search=ArtifactSearchDeclaration( + version="1", indexes=(*families[0].search.indexes, VectorIndex( + name="{index_name}", projection_version="1", source="content", + vector_field="vector", profile_field="profile", profile=VECTOR_PROFILE, + )), modes=("text", "vector", "hybrid"), default_mode="text", +)),) +''' + ) + return plugin + + async def run(): + async with running_server(tmp_path) as client: + scope = await create_scope(client, "profile-admission") + first = deploy("1", "features-v1") + state = await reload_plugin(client, tmp_path, first, 0) + # No content exists yet; the declaration is nevertheless already fixed. + second = deploy("2", "features-v2") + publish_plugin(tmp_path, second) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await client.get(REGISTRATION)).json() == state + publish_plugin(tmp_path, first) + await enable_all(client) + state = await plugin_state(client, "enabled") + await capture(client, scope, "original") + items = await artifacts(client, scope, 1) + await _wait_for_search_ready(client, scope, "Outage") + state = await change(client, "disable", state["binding_generation"], "disabled") + before = await _search(client, scope, "Outage") + vector_request = { + "mode": "vector", + "query_vector": { + "profile": before["results"][0]["content"]["profile"], + "vector": [1.0, 0.0], + }, + } + url = f"/v1/scopes/{scope}/artifacts/incident-report/search" + vector_before = await client.post(url, json=vector_request) + assert vector_before.status_code == 200, vector_before.text + assert vector_before.json()["results"] + # Neither profile, name, projection version, schema identity nor index membership may change. + edits = ( + ("features-v1", "features-v2"), + ('name="visual"', 'name="renamed"'), + ('projection_version="1"', 'projection_version="2"'), + ('schema_version="1"', 'schema_version="2"'), + ('version="1", indexes=', 'version="2", indexes='), + ("indexes=(*families[0].search.indexes, VectorIndex(", "indexes=(VectorIndex("), + ) + for number, (old, new) in enumerate(edits, start=3): + replacement = deploy(str(number), "features-v1") + entry = replacement / "plugin.py" + assert old in entry.read_text() + code = entry.read_text().replace(old, new) + if old.startswith("indexes="): + code = code.replace(")), modes=", "),), modes=") + code = code.replace( + 'modes=("text", "vector", "hybrid"), default_mode="text"', + 'modes=("vector",), default_mode="vector"', + ) + entry.write_text(code) + publish_plugin(tmp_path, replacement) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await client.get(REGISTRATION)).json() == state + assert await _search(client, scope, "Outage") == before + unchanged = await client.post(url, json=vector_request) + assert unchanged.status_code == 200, unchanged.text + assert unchanged.json() == vector_before.json() + assert await artifacts(client, scope, 1) == items + policy = deploy("query-policy", "features-v1") + entry = policy / "plugin.py" + entry.write_text( + entry.read_text().replace( + 'modes=("text", "vector", "hybrid"), default_mode="text",', + 'modes=("text", "vector"), default_mode="vector", ' + 'defaults={"text": {"text": ("lexical",)}, "vector": {"vector": ("visual",)}}, ' + "candidate_limit=17, rrf_k=11, text_weight=2, vector_weight=3,", + ) + ) + state = await reload_plugin(client, tmp_path, policy, state["binding_generation"]) + default_vector = {"query_vector": vector_request["query_vector"]} + result = await client.post(url, json=default_vector) + assert result.status_code == 200, result.text + assert result.json() == vector_before.json() + assert (await client.post(url, json={"mode": "text", "query": "Outage"})).json() == before + assert (await client.post(url, json={"mode": "hybrid", "query": "Outage"})).status_code == 422 + assert await artifacts(client, scope, 1) == items + # Invalid policy must not replace the last accepted configuration. + entry.write_text(entry.read_text().replace('("visual",)}', '("missing",)}')) + publish_plugin(tmp_path, policy) + invalid = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert invalid.status_code == 422, invalid.text + assert (await client.get(REGISTRATION)).json() == state + assert (await client.post(url, json=default_vector)).json() == vector_before.json() + state = await change(client, "unload", state["binding_generation"], "unloaded") + async with running_server(tmp_path) as client: + # Unload clears the runtime definition, but the persisted contract still rejects changes. + publish_plugin(tmp_path, second) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await client.get(REGISTRATION)).json() == state + assert (await client.post(url, json={"mode": "text", "query": "Outage"})).json() == before + assert (await client.post(url, json=default_vector)).json() == vector_before.json() + assert (await client.post(url, json={"mode": "hybrid", "query": "Outage"})).status_code == 422 + compatible = deploy("code-only", "features-v1") + state = await reload_plugin(client, tmp_path, compatible, state["binding_generation"]) + assert await _search(client, scope, "Outage") == before + assert (await client.get(REGISTRATION)).json()["binding_generation"] == state["binding_generation"] + + asyncio.run(run()) diff --git a/tests/e2e/test_artifact_plugins.py b/tests/e2e/test_artifact_plugins.py new file mode 100644 index 0000000000..95ba908d24 --- /dev/null +++ b/tests/e2e/test_artifact_plugins.py @@ -0,0 +1,1014 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Use a delivered plugin through the installed Server CLI and public HTTP APIs.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import socket +import sys +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import httpx +import yaml + +EXAMPLE = Path(__file__).resolve().parents[2] / "examples/artifact_plugins/incident_report" +SUMMARY_EXAMPLE = EXAMPLE.parent / "ticket_summary" +ADMIN_AUTHORIZATION_VALUE = "artifact-plugin-e2e-admin" +REGISTRATION = "/v1/admin/artifact-plugins/incident-reports" +SUMMARY_REGISTRATION = "/v1/admin/artifact-plugins/ticket-summaries" + + +def deploy_plugin(source: Path, directory: Path, scope_ids: list[str] | None, **parameters: Any) -> Path: + """Prepare a deployed version outside the discovery directory.""" + + shutil.copytree(source, directory, ignore=shutil.ignore_patterns("__pycache__")) + path = directory / "plugin.yml" + manifest = yaml.safe_load(path.read_text()) + manifest.update(parameters) + if scope_ids is None: + manifest.pop("scope_ids", None) + else: + manifest["scope_ids"] = scope_ids + path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8") + return directory + + +def publish_plugin(directory: Path, plugin: Path, binding_name: str | None = None) -> Path: + """Switch a user's discovery entry without changing any previous version.""" + + if binding_name is None: + binding_name = yaml.safe_load((plugin / "plugin.yml").read_text())["binding_name"] + root = directory / "plugins" + root.mkdir(exist_ok=True) + entry = root / binding_name + entry.unlink(missing_ok=True) + entry.symlink_to(plugin, target_is_directory=True) + return entry + + +async def reload_plugin( + client: httpx.AsyncClient, + directory: Path, + plugin: Path, + generation: int, + phase: str = "disabled", + registration: str = REGISTRATION, +) -> dict[str, Any]: + publish_plugin(directory, plugin, registration.rsplit("/", 1)[1]) + response = await client.post(f"{registration}/reload", json={"expected_generation": generation}) + assert response.status_code == 202, response.text + assert response.headers["location"] == registration + return await plugin_state(client, phase, registration) + + +async def enable_all( + client: httpx.AsyncClient, *, expected_failures: frozenset[str] = frozenset() +) -> dict[str, dict[str, Any]]: + response = await client.post("/v1/admin/artifact-plugins/enable-all") + assert response.status_code == 200, response.text + items = response.json()["items"] + failed = frozenset(item["binding_name"] for item in items if item["outcome"] == "failed") + assert failed == expected_failures, response.text + assert all(item["status_url"] == f"/v1/admin/artifact-plugins/{item['binding_name']}" for item in items) + return {item["binding_name"]: item for item in items} + + +@asynccontextmanager +async def running_server( + directory: Path, + *, + envfile: Path | None = None, + directory_from_env: bool = True, + database_url: str | None = None, + database_settings: dict[str, Any] | None = None, +): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + env = {key: value for key, value in os.environ.items() if not key.startswith("POWERCONTEXT_SERVER_")} + env.update({ + "POWERCONTEXT_SERVER_DATABASE": json.dumps( + database_settings + or { + "kind": "sqlite", + "url": database_url or f"sqlite+aiosqlite:///{directory / 'server.db'}", + } + ), + "POWERCONTEXT_SERVER_RUNTIME": json.dumps({"artifact_processing_families": []}), + "POWERCONTEXT_SERVER_WORKSPACE": str(directory), + "POWERCONTEXT_HOME": str(directory / "home"), + "POWERCONTEXT_SERVER_AUTH_ENABLED": "true", + "POWERCONTEXT_SERVER_AUTH_TOKEN": ADMIN_AUTHORIZATION_VALUE, + "POWERCONTEXT_SERVER_ACCESS_MODE": "enforced", + "POWERCONTEXT_SERVER_MCP_ENABLED": "false", + "POWERCONTEXT_SERVER_METRICS_ENABLED": "true", + "POWERCONTEXT_SERVER_DASHBOARD_ENABLED": "false", + }) + (directory / "plugins").mkdir(exist_ok=True) + if directory_from_env: + env["POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR"] = str(directory / "plugins") + envfile_args = ["--no-env-file"] if envfile is None else ["--env-file", str(envfile)] + entrypoint = Path(sys.executable).with_name("powercontext") + with (directory / "server.log").open("a") as log: + process = await asyncio.create_subprocess_exec( + sys.executable, + str(entrypoint), + "server", + "run", + *envfile_args, + "--host", + "127.0.0.1", + "--port", + str(port), + cwd=directory, + env=env, + stdout=log, + stderr=log, + ) + try: + async with httpx.AsyncClient( + base_url=f"http://127.0.0.1:{port}", + headers={"Authorization": f"Bearer {ADMIN_AUTHORIZATION_VALUE}"}, + timeout=20, + ) as client: + async with asyncio.timeout(40): + while True: + assert process.returncode is None, (directory / "server.log").read_text() + try: + response = await client.get("/health/ready") + if response.status_code == 200: + break + except httpx.ConnectError: + pass + await asyncio.sleep(0.1) + yield client + finally: + if process.returncode is None: + process.terminate() + try: + await asyncio.wait_for(process.wait(), 20) + except TimeoutError: + process.kill() + await process.wait() + + +async def create_scope(client: httpx.AsyncClient, key: str) -> str: + response = await client.post( + "/v1/scopes", json={"title": key, "summary": "Artifact plugin acceptance", "idempotency_key": key} + ) + assert response.status_code == 201, response.text + return response.json()["scope_id"] + + +async def capture(client: httpx.AsyncClient, scope: str, source_id: str) -> None: + response = await client.post( + "/v1/sources/content", + json={ + "scope_id": scope, + "source_id": source_id, + "content": f"Outage {source_id}: database recovered after retry.", + }, + ) + assert response.status_code == 202, response.text + + +async def plugin_state(client: httpx.AsyncClient, phase: str, registration: str = REGISTRATION) -> dict[str, Any]: + async with asyncio.timeout(40): + while True: + response = await client.get(registration) + assert response.status_code == 200, response.text + state = response.json() + assert state["phase"] != "failed" or phase == "failed", state + if state["phase"] == phase: + return state + await asyncio.sleep(0.1) + + +async def artifacts( + client: httpx.AsyncClient, scope: str, count: int, family: str = "incident-report" +) -> list[dict[str, Any]]: + async with asyncio.timeout(45): + while True: + response = await client.get(f"/v1/scopes/{scope}/artifacts/{family}") + assert response.status_code == 200, response.text + items = response.json()["items"] + if len(items) == count: + return items + await asyncio.sleep(0.2) + + +async def query(client: httpx.AsyncClient, scope: str, family: str = "incident-report") -> dict[str, Any]: + response = await client.post(f"/v1/scopes/{scope}/artifacts/{family}/search", json={"query": "Outage"}) + assert response.status_code == 200, response.text + return response.json() + + +async def _wait_for_search_ready(client, scope, query): + async with asyncio.timeout(30): + while True: + response = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": query}) + if response.status_code != 503: + assert response.status_code == 200, response.text + return + assert response.json()["error"]["code"] == "artifact_search_unavailable", response.text + await asyncio.sleep(0.1) + + +async def change( + client: httpx.AsyncClient, action: str, generation: int, phase: str, registration: str = REGISTRATION +) -> dict[str, Any]: + response = await client.post(f"{registration}/{action}", json={"expected_generation": generation}) + assert response.status_code == 202, response.text + assert response.headers["location"] == registration + return await plugin_state(client, phase, registration) + + +async def wait_for_worker_metric(client: httpx.AsyncClient, metric: str, family: str = "incident-report") -> None: + async with asyncio.timeout(40): + while True: + response = await client.get("/metrics") + assert response.status_code == 200, response.text + if any( + f'family="{family}"' in line and float(line.rsplit(" ", 1)[1]) > 0 + for line in response.text.splitlines() + if line.startswith(f"powercontext_server_artifact_processing_{metric}{{") + ): + return + await asyncio.sleep(0.2) + + +async def worker_metric(client: httpx.AsyncClient, metric: str, family: str) -> float: + response = await client.get("/metrics") + assert response.status_code == 200, response.text + values = [ + float(line.rsplit(" ", 1)[1]) + for line in response.text.splitlines() + if line.startswith(f'powercontext_server_artifact_processing_{metric}{{family="{family}"}} ') + ] + assert len(values) == 1, response.text + return values[0] + + +async def contents(client: httpx.AsyncClient, scope: str, family: str, count: int) -> list[dict[str, Any]]: + result = [] + for item in await artifacts(client, scope, count, family): + response = await client.get(f"/v1/scopes/{scope}/artifacts/{family}/{item['artifact_id']}") + assert response.status_code == 200, response.text + result.append(response.json()) + return result + + +def test_strict_plugin_configuration_survives_generation_reload_and_restart(tmp_path: Path) -> None: + asyncio.run(_use_strict_plugin_configuration(tmp_path)) + + +async def _use_strict_plugin_configuration(tmp_path: Path) -> None: + plugin = deploy_plugin(EXAMPLE, tmp_path / "strict-v1", None) + entry = plugin / "plugin.py" + entry.write_text( + entry + .read_text() + .replace("import hashlib", "import hashlib\nfrom datetime import date\nfrom enum import Enum") + .replace('ConfigDict(extra="forbid", frozen=True)', 'ConfigDict(extra="forbid", frozen=True, strict=True)') + .replace( + "class Config(BaseModel):", + 'class Mode(str, Enum):\n NORMAL = "normal"\n URGENT = "urgent"\n\n\nclass Config(BaseModel):', + ) + .replace( + ' title_prefix: str = "Incident report"', + ' mode: Mode = Mode.NORMAL\n observed_at: date = date(2026, 10, 10)\n labels: tuple[str, ...] = ("初始",)\n title_prefix: str = "Incident report"', + ) + .replace( + " return IncidentReportPipeline(config)", + ' prefix = f"{config.mode.value}/{config.observed_at.isoformat()}/{config.labels}"\n return IncidentReportPipeline(config.model_copy(update={"title_prefix": prefix}))', + ) + ) + async with running_server(tmp_path) as client: + scope = await create_scope(client, "strict-configuration") + await reload_plugin(client, tmp_path, plugin, 0) + await change(client, "enable", 1, "enabled") + await capture(client, scope, "before-restart") + reports = await contents(client, scope, "incident-report", 1) + assert reports[0]["content"]["title"].startswith("normal/2026-10-10/('初始',):") + async with running_server(tmp_path) as client: + state = await plugin_state(client, "enabled") + await capture(client, scope, "after-restart") + reports = await contents(client, scope, "incident-report", 2) + assert all(item["content"]["title"].startswith("normal/2026-10-10/('初始',):") for item in reports) + replacement = deploy_plugin( + plugin, + tmp_path / "strict-v2", + None, + version="2", + config={"mode": "urgent", "observed_at": "2026-10-11", "labels": ["更新"]}, + ) + await reload_plugin(client, tmp_path, replacement, state["binding_generation"], "enabled") + await capture(client, scope, "after-reload") + reports = await contents(client, scope, "incident-report", 3) + assert next(item for item in reports if item["sources"][0]["source_id"] == "after-reload")["content"][ + "title" + ].startswith("urgent/2026-10-11/('更新',):") + + +def test_plugin_cannot_register_the_builtin_atomic_memory_family(tmp_path: Path) -> None: + asyncio.run(_reject_builtin_atomic_memory_plugin(tmp_path)) + + +async def _reject_builtin_atomic_memory_plugin(tmp_path: Path) -> None: + plugin = deploy_plugin(EXAMPLE, tmp_path / "builtin-family", None) + entry = plugin / "plugin.py" + entry.write_text(entry.read_text().replace('"incident-report"', '"atomic-memory"')) + async with running_server(tmp_path) as client: + publish_plugin(tmp_path, plugin) + response = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 0}) + assert response.status_code == 422, response.text + response = await client.get("/v1/admin/artifact-plugins") + assert response.status_code == 200, response.text + assert response.json()["items"] == [] + + +def test_in_memory_database_rejects_plugin_enable_without_changing_registration(tmp_path: Path) -> None: + asyncio.run(_reject_in_memory_plugin_generation(tmp_path)) + + +async def _reject_in_memory_plugin_generation(tmp_path: Path) -> None: + plugin = deploy_plugin(EXAMPLE, tmp_path / "memory-plugin", None) + publish_plugin(tmp_path, plugin) + async with running_server(tmp_path, database_url="sqlite+aiosqlite:///:memory:") as client: + state = await plugin_state(client, "disabled") + scope = await create_scope(client, "in-memory-input") + await capture(client, scope, "pending-input") + response = await client.post( + f"{REGISTRATION}/enable", json={"expected_generation": state["binding_generation"]} + ) + assert response.status_code == 422, response.text + assert await plugin_state(client, "disabled") == state + item = (await enable_all(client, expected_failures=frozenset({"incident-reports"})))["incident-reports"] + assert item["outcome"] == "failed" + assert item["error"] == "invalid_plugin" + assert item["state"] == state + assert await artifacts(client, scope, 0) == [] + + +def test_plugin_manifests_reject_enable_fields_without_replacing_saved_definition(tmp_path: Path) -> None: + asyncio.run(_reject_manifest_enable_fields(tmp_path)) + + +async def _reject_manifest_enable_fields(tmp_path: Path) -> None: + report = deploy_plugin(EXAMPLE, tmp_path / "report-v1", None, config={"title_prefix": "Saved"}) + publish_plugin(tmp_path, report) + invalid_registrations = [] + for field in ("enable", "enabled"): + binding_name = f"invalid-{field}" + plugin = deploy_plugin( + SUMMARY_EXAMPLE, + tmp_path / binding_name, + None, + binding_name=binding_name, + **{field: True}, + ) + publish_plugin(tmp_path, plugin) + invalid_registrations.append((field, binding_name)) + + async with running_server(tmp_path) as client: + state = await plugin_state(client, "disabled") + response = await client.get("/v1/admin/artifact-plugins") + assert response.status_code == 200, response.text + assert response.json()["items"] == [state] + for field, binding_name in invalid_registrations: + registration = f"/v1/admin/artifact-plugins/{binding_name}" + assert (await client.get(registration)).status_code == 404 + for value in (True, False): + candidate = deploy_plugin( + SUMMARY_EXAMPLE, + tmp_path / f"{binding_name}-{value}", + None, + binding_name=binding_name, + **{field: value}, + ) + publish_plugin(tmp_path, candidate) + rejected = await client.post(f"{registration}/reload", json={"expected_generation": 0}) + assert rejected.status_code == 422, rejected.text + assert (await client.get(registration)).status_code == 404 + + assert set(await enable_all(client)) == {"incident-reports"} + state = await plugin_state(client, "enabled") + scope = await create_scope(client, "manifest-validation") + await capture(client, scope, "before-invalid-update") + await artifacts(client, scope, 1) + for field in ("enable", "enabled"): + for value in (True, False): + candidate = deploy_plugin( + EXAMPLE, + tmp_path / f"report-{field}-{value}", + None, + config={"title_prefix": "Rejected"}, + **{field: value}, + ) + publish_plugin(tmp_path, candidate) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 422, rejected.text + assert await plugin_state(client, "enabled") == state + await capture(client, scope, "after-invalid-update") + results = await contents(client, scope, "incident-report", 2) + assert all(item["content"]["title"].startswith("Saved:") for item in results) + + +def test_user_plugin_through_server_hot_management_and_restart(tmp_path: Path) -> None: + asyncio.run(_use_plugin_through_server(tmp_path)) + + +async def _use_plugin_through_server(tmp_path: Path) -> None: + # Existing Scopes and inputs are created through the delivered HTTP API. + async with running_server(tmp_path) as client: + scope = await create_scope(client, "reports") + other_scope = await create_scope(client, "other") + await capture(client, scope, "ticket-1") + plugin = deploy_plugin(EXAMPLE, tmp_path / "incident-report-v1", [scope], config={"title_prefix": "Report"}) + factory_signal = tmp_path / "pipeline-created" + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text().replace( + " return IncidentReportPipeline(config)", + f" from pathlib import Path\n Path({str(factory_signal)!r}).write_text('created')\n return IncidentReportPipeline(config)", + ) + ) + publish_plugin(tmp_path, plugin) + broken = deploy_plugin( + SUMMARY_EXAMPLE, tmp_path / "broken-v1", [scope], binding_name="broken-summary", config={"unknown": True} + ) + publish_plugin(tmp_path, broken) + unregistered = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "unregistered-plugin", [scope]) + # Neither version directories outside the root nor nested entries are discovered. + nested = tmp_path / "plugins" / "nested" + nested.mkdir() + (nested / "ticket-summaries").symlink_to(unregistered, target_is_directory=True) + envfile = tmp_path / "server.env" + envfile.write_text(f"POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR={tmp_path / 'plugins'}\n") + async with running_server(tmp_path, envfile=envfile, directory_from_env=False) as client: + state = await plugin_state(client, "disabled") + assert state["binding_generation"] == 1 + registrations = (await client.get("/v1/admin/artifact-plugins")).json()["items"] + assert [item["binding_name"] for item in registrations] == ["incident-reports"] + await asyncio.sleep(2) + assert not factory_signal.exists(), "Loading a declaration must not construct the generation pipeline" + assert await artifacts(client, scope, 0) == [] + metadata = (await client.get("/v1/capabilities")).json() + assert "incident-report" in metadata["artifact_families"] + schema = next(item for item in metadata["custom_artifact_schemas"] if item["family"] == "incident-report") + assert schema["schema_version"] == "1" + assert schema["search"]["indexes"][0]["fields"] == ["title", "evidence_text"] + denied = await client.post("/v1/admin/artifact-plugins/enable-all", headers={"Authorization": ""}) + assert denied.status_code == 401 + obsolete = await client.post( + f"{REGISTRATION}/reload", + json={ + "expected_generation": 1, + "directory": str(plugin), + "scope_ids": [scope], + "config": {}, + "enabled": True, + }, + ) + assert obsolete.status_code == 422, obsolete.text + assert (await client.put(REGISTRATION, json={"expected_generation": 1})).status_code == 405 + accepted = await enable_all(client) + assert accepted["incident-reports"]["outcome"] == "accepted" + state = await plugin_state(client, "enabled") + assert state["binding_generation"] == 2 + await artifacts(client, scope, 1) + assert factory_signal.exists() + assert (await enable_all(client))["incident-reports"]["outcome"] == "skipped" + await capture(client, scope, "ticket-2") + await capture(client, other_scope, "outside-scope") + items = await artifacts(client, scope, 2) + assert {item["revision"] for item in items} == {1} + page = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report", params={"limit": 1}) + assert page.status_code == 200, page.text + next_page = await client.get( + f"/v1/scopes/{scope}/artifacts/incident-report", params={"limit": 1, "cursor": page.json()["next_cursor"]} + ) + assert next_page.status_code == 200, next_page.text + assert {page.json()["items"][0]["artifact_id"], next_page.json()["items"][0]["artifact_id"]} == { + item["artifact_id"] for item in items + } + for item in items: + result = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{item['artifact_id']}") + assert result.status_code == 200, result.text + artifact = result.json() + assert artifact["schema_version"] == "1" + assert artifact["content"]["title"].startswith("Report: ticket-") + assert artifact["content"]["evidence_text"].startswith("Outage ticket-") + assert artifact["sources"] == [ + {"source_type": "content", "source_id": artifact["content"]["title"].split(": ")[1]} + ] + cached = await client.get( + f"/v1/scopes/{scope}/artifacts/incident-report/{item['artifact_id']}", + headers={"If-None-Match": result.headers["etag"]}, + ) + assert cached.status_code == 304 + assert len((await query(client, scope))["results"]) == 2 + assert (await query(client, other_scope))["results"] == [] + await capture(client, scope, "ticket-1") + stale = await client.post(f"{REGISTRATION}/disable", json={"expected_generation": 99}) + assert stale.status_code == 409 + state = await change(client, "disable", 2, "disabled") + await capture(client, scope, "ticket-3") + assert len((await query(client, scope))["results"]) == 2 + assert "enabled" not in yaml.safe_load((plugin / "plugin.yml").read_text()) + replacement = deploy_plugin( + EXAMPLE, tmp_path / "incident-report-v2", [scope], version="2", config={"title_prefix": "Replacement"} + ) + publish_plugin(tmp_path, replacement) + # Environment variables take priority over an envfile. Scanning cannot reset a saved disable or update its code. + envfile.write_text(f"POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR={tmp_path / 'wrong-discovery-root'}\n") + async with running_server(tmp_path, envfile=envfile) as client: + state = await plugin_state(client, "disabled") + assert state["binding_generation"] == 3 + assert len((await query(client, scope))["results"]) == 2 + assert len((await client.get("/v1/admin/artifact-plugins")).json()["items"]) == 1 + state = await change(client, "enable", state["binding_generation"], "enabled") + assert state["binding_generation"] == 4 + results = await contents(client, scope, "incident-report", 3) + assert all(item["content"]["title"].startswith("Report:") for item in results) + # Enabled registrations resume without another batch enable and keep the resolved saved version. + async with running_server(tmp_path) as client: + state = await plugin_state(client, "enabled") + assert state["binding_generation"] > 4 + await capture(client, scope, "ticket-4") + results = await contents(client, scope, "incident-report", 4) + assert all(item["content"]["title"].startswith("Report:") for item in results) + # Explicit reload picks up the switched entry and keeps this plugin enabled. + state = await reload_plugin(client, tmp_path, replacement, state["binding_generation"], "enabled") + unchanged = await reload_plugin(client, tmp_path, replacement, state["binding_generation"], "enabled") + assert unchanged["binding_generation"] == state["binding_generation"] + unloaded = await change(client, "unload", state["binding_generation"], "unloaded") + shutil.rmtree(plugin) + # Startup must not load even a valid discovery entry for an unloaded registration. + async with running_server(tmp_path) as client: + state = await plugin_state(client, "unloaded") + assert state["binding_generation"] == unloaded["binding_generation"] + assert (await enable_all(client))["incident-reports"]["outcome"] == "skipped" + assert len((await query(client, scope))["results"]) == 4 + await capture(client, scope, "ticket-5") + assert len(await artifacts(client, scope, 4)) == 4 + for item in results: + result = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{item['artifact_id']}") + assert result.status_code == 200, result.text + state = await reload_plugin(client, tmp_path, replacement, state["binding_generation"]) + assert len((await query(client, scope))["results"]) == 4 + await change(client, "enable", state["binding_generation"], "enabled") + results = await contents(client, scope, "incident-report", 5) + assert ( + next(item for item in results if item["sources"][0]["source_id"] == "ticket-5")["content"]["title"] + == "Replacement: ticket-5" + ) + + +def test_generation_failure_retries_uncommitted_inputs_after_config_update(tmp_path: Path) -> None: + asyncio.run(_recover_generation_failure(tmp_path)) + + +async def _recover_generation_failure(tmp_path: Path) -> None: + code = tmp_path / "user-plugin-code" + shutil.copytree(EXAMPLE, code, ignore=shutil.ignore_patterns("__pycache__")) + entry = code / "plugin.py" + entry.write_text( + entry + .read_text() + .replace( + ' title_prefix: str = "Incident report"', + ' deployment_label: str = Field(alias="label")\n title_prefix: str = "Incident report"', + ) + .replace( + " return IncidentReportPipeline(config)", + ' if config.deployment_label != "recovery":\n raise ValueError("Invalid deployment label")\n return IncidentReportPipeline(config)', + ) + ) + async with running_server(tmp_path) as client: + scope = await create_scope(client, "reports") + replacement_scope = await create_scope(client, "replacement") + plugin = deploy_plugin( + code, tmp_path / "plugin-v1", [scope], config={"fail_on_text": "Outage", "label": "recovery"} + ) + await capture(client, scope, "failed-ticket") + await reload_plugin(client, tmp_path, plugin, 0) + await change(client, "enable", 1, "enabled") + # The product exposes Worker failures through its authenticated metrics. + await wait_for_worker_metric(client, "failed_total") + assert await artifacts(client, scope, 0) == [] + invalid_directory = deploy_plugin(plugin, tmp_path / "invalid-config", [scope], config={"unknown": True}) + publish_plugin(tmp_path, invalid_directory, "incident-reports") + invalid = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert invalid.status_code == 422, invalid.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + invalid_entry = deploy_plugin(plugin, tmp_path / "invalid-entry", [scope]) + entry = invalid_entry / "plugin.py" + entry.write_text( + entry.read_text().replace( + "async def create_pipeline(ctx: GenerationContext, config: Config)", + "async def create_pipeline(config: Config)", + ) + ) + publish_plugin(tmp_path, invalid_entry, "incident-reports") + invalid = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert invalid.status_code == 422, invalid.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + mismatch = deploy_plugin(plugin, tmp_path / "wrong-binding", [scope], binding_name="another-binding") + publish_plugin(tmp_path, mismatch, "incident-reports") + invalid = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert invalid.status_code == 422, invalid.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + incompatible = deploy_plugin(plugin, tmp_path / "incompatible-schema", [scope]) + entry = incompatible / "plugin.py" + entry.write_text(entry.read_text().replace("Field(min_length=1)", "Field(min_length=2)")) + publish_plugin(tmp_path, incompatible, "incident-reports") + conflict = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert conflict.status_code == 409, conflict.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + replacement = deploy_plugin( + plugin, tmp_path / "plugin-v2", [replacement_scope], version="2", config={"label": "recovery"} + ) + await reload_plugin(client, tmp_path, replacement, 2, "enabled") + await capture(client, replacement_scope, "replacement-ticket") + await artifacts(client, replacement_scope, 1) + await wait_for_worker_metric(client, "completed_total") + assert await artifacts(client, scope, 0) == [] + metrics = await client.get("/metrics") + assert metrics.status_code == 200, metrics.text + failures = [ + float(line.rsplit(" ", 1)[1]) + for line in metrics.text.splitlines() + if line.startswith('powercontext_server_artifact_processing_failed_total{family="incident-report"}') + ] + assert failures == [0.0], "Removed Scopes must not keep dispatching failed Workers" + # Re-adding a Scope resumes its retained input instead of losing the failed window. + restored = deploy_plugin( + replacement, tmp_path / "plugin-v3", [scope, replacement_scope], version="3", config={"label": "recovery"} + ) + await reload_plugin(client, tmp_path, restored, 3, "enabled") + items = await artifacts(client, scope, 1) + assert items[0]["revision"] == 1 + # First publication is asynchronous even after generation has committed. + await _wait_for_search_ready(client, scope, "Outage") + assert len((await query(client, scope))["results"]) == 1 + + +def test_disabling_an_active_worker_reclaims_it_before_reenabling(tmp_path: Path) -> None: + asyncio.run(_disable_active_generation(tmp_path)) + + +async def _disable_active_generation(tmp_path: Path) -> None: + code = tmp_path / "user-plugin-code" + shutil.copytree(EXAMPLE, code, ignore=shutil.ignore_patterns("__pycache__")) + entry = code / "plugin.py" + entry.write_text( + entry + .read_text() + .replace("import hashlib", "import asyncio\nimport hashlib") + .replace( + ' title_prefix: str = "Incident report"', + ' block_generation: bool = False\n title_prefix: str = "Incident report"', + ) + + """ + +class SummaryPipeline: + def __init__(self, ctx, config): + self.ctx = ctx + self.config = config + + async def generate(self, batch): + if self.config.block_generation: + await asyncio.Future() + previous = await self.ctx.read_latest("incident-report", "latest-report") + old_text = "" if previous is None else previous.content["evidence_text"] + "\\n" + return [ArtifactDraft( + family="incident-report", schema_version="1", artifact_id="latest-report", + expected_revision=0 if previous is None else previous.ref.revision, + content={"title": self.config.title_prefix, + "evidence_text": old_text + "\\n".join(TextEvidence.model_validate(source.value).content for source in batch.sources)}, + source_refs=tuple(source.ref for source in batch.sources), + artifact_refs=() if previous is None else (previous.ref,), + )] + +async def create_pipeline(ctx, config): + return SummaryPipeline(ctx, config) +""" + ) + async with running_server(tmp_path) as client: + scope = await create_scope(client, "reports") + plugin = deploy_plugin(code, tmp_path / "plugin-v1", [scope], config={"block_generation": True}) + await capture(client, scope, "active-ticket") + await reload_plugin(client, tmp_path, plugin, 0) + await change(client, "enable", 1, "enabled") + await wait_for_worker_metric(client, "used_workers") + await change(client, "disable", 2, "disabled") + assert await artifacts(client, scope, 0) == [] + replacement = deploy_plugin( + plugin, + tmp_path / "plugin-v2", + [scope], + version="2", + config={"block_generation": False, "title_prefix": "Replacement"}, + ) + state = await reload_plugin(client, tmp_path, replacement, 3) + assert await artifacts(client, scope, 0) == [] + await change(client, "enable", state["binding_generation"], "enabled") + items = await artifacts(client, scope, 1) + response = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{items[0]['artifact_id']}") + assert response.status_code == 200, response.text + assert response.json()["revision"] == 1 + assert response.json()["content"]["title"] == "Replacement" + await capture(client, scope, "update-ticket") + async with asyncio.timeout(45): + while True: + response = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/latest-report") + assert response.status_code == 200, response.text + if response.json()["revision"] == 2: + break + await asyncio.sleep(0.2) + result = response.json() + assert "active-ticket" in result["content"]["evidence_text"] + assert "update-ticket" in result["content"]["evidence_text"] + assert result["sources"] == [{"source_type": "content", "source_id": "update-ticket"}] + assert result["artifacts"] == [{"family": "incident-report", "artifact_id": "latest-report", "revision": 1}] + assert (await query(client, scope))["results"][0]["revision"] == 2 + previous = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/latest-report/revisions/1") + assert previous.status_code == 200, previous.text + assert "update-ticket" not in previous.json()["content"]["evidence_text"] + + +def test_multiple_plugin_manifests_apply_independent_configuration(tmp_path: Path) -> None: + asyncio.run(_use_independent_plugins(tmp_path)) + + +async def _use_independent_plugins(tmp_path: Path) -> None: + summary_registration = "/v1/admin/artifact-plugins/ticket-summaries" + async with running_server(tmp_path) as client: + report_scope = await create_scope(client, "reports") + other_report_scope = await create_scope(client, "other-reports") + summary_scope = await create_scope(client, "summaries") + report = deploy_plugin( + EXAMPLE, + tmp_path / "report-v1", + [report_scope, other_report_scope], + max_sources=1, + processing={"interval_seconds": 0.5, "max_workers": 2, "timeout_seconds": 60}, + config={"title_prefix": "独立报告"}, + ) + # This user pipeline accepts a single evidence item per invocation. + entry = report / "plugin.py" + entry.write_text( + entry.read_text().replace( + " drafts = []", + ' if len(batch.sources) != 1:\n raise ValueError("Expected one ticket per report invocation")\n drafts = []', + ) + ) + summary = deploy_plugin( + SUMMARY_EXAMPLE, + tmp_path / "summary-v1", + [summary_scope], + max_sources=2, + processing={"interval_seconds": 15, "max_workers": 1, "timeout_seconds": 90}, + config={"language": "Français"}, + ) + for scope in (report_scope, summary_scope): + for number in range(3): + await capture(client, scope, f"ticket-{number}") + await capture(client, other_report_scope, "another-ticket") + assert (await client.get("/v1/admin/artifact-plugins")).json()["items"] == [] + for registration, directory in ((REGISTRATION, report), (summary_registration, summary)): + await reload_plugin(client, tmp_path, directory, 0, registration=registration) + assert await artifacts(client, report_scope, 0) == [] + assert await artifacts(client, summary_scope, 0, "ticket-summary") == [] + # A temporarily unavailable deployed version fails independently; successful items stay accepted. + unavailable = summary.with_name("summary-v1-unavailable") + summary.rename(unavailable) + results = await enable_all(client, expected_failures=frozenset({"ticket-summaries"})) + assert results["incident-reports"]["outcome"] == "accepted" + assert results["ticket-summaries"]["outcome"] == "failed" + assert results["ticket-summaries"]["error"] == "invalid_plugin" + assert (await plugin_state(client, "disabled", summary_registration))["binding_generation"] == 1 + await plugin_state(client, "enabled") + summary = unavailable.rename(summary) + results = await enable_all(client) + assert results["incident-reports"]["outcome"] == "skipped" + assert results["ticket-summaries"]["outcome"] == "accepted" + await plugin_state(client, "enabled", summary_registration) + assert all(item["outcome"] == "skipped" for item in (await enable_all(client)).values()) + started = asyncio.get_running_loop().time() + assert await worker_metric(client, "max_workers", "incident-report") == 2 + assert await worker_metric(client, "max_workers", "ticket-summary") == 1 + # No second automatic window may be admitted before this binding's interval. + # The first window can run immediately and must contain at most two Sources. + while asyncio.get_running_loop().time() < started + 14: + response = await client.get(f"/v1/scopes/{summary_scope}/artifacts/ticket-summary") + assert response.status_code == 200, response.text + assert len(response.json()["items"]) <= 1, "The summary binding ignored its 15-second interval" + await asyncio.sleep(0.2) + reports = await contents(client, report_scope, "incident-report", 3) + assert all(item["content"]["title"].startswith("独立报告:") for item in reports) + assert all(len(item["sources"]) == 1 for item in reports) + await artifacts(client, other_report_scope, 1) + summaries = await contents(client, summary_scope, "ticket-summary", 2) + assert sorted(len(item["sources"]) for item in summaries) == [1, 2] + assert all(item["content"]["language"] == "Français" for item in summaries) + assert {source["source_id"] for item in summaries for source in item["sources"]} == { + "ticket-0", + "ticket-1", + "ticket-2", + } + assert (await query(client, report_scope, "ticket-summary"))["results"] == [] + assert (await query(client, summary_scope))["results"] == [] + assert "enabled" not in yaml.safe_load((report / "plugin.yml").read_text()) + assert "enabled" not in yaml.safe_load((summary / "plugin.yml").read_text()) + replacement = deploy_plugin( + summary, + tmp_path / "summary-v2", + [summary_scope], + version="2", + max_sources=1, + processing={"interval_seconds": 0.5, "max_workers": 2, "timeout_seconds": 60}, + config={"language": "Deutsch"}, + ) + await reload_plugin(client, tmp_path, replacement, 2, "enabled", summary_registration) + for number in (3, 4): + await capture(client, summary_scope, f"ticket-{number}") + summaries = await contents(client, summary_scope, "ticket-summary", 4) + updated = [item for item in summaries if item["content"]["language"] == "Deutsch"] + assert len(updated) == 2 + assert all(len(item["sources"]) == 1 for item in updated) + assert await worker_metric(client, "max_workers", "ticket-summary") == 2 + assert await worker_metric(client, "max_workers", "incident-report") == 2 + await change(client, "unload", 3, "unloaded", summary_registration) + await capture(client, report_scope, "report-after-summary-unload") + await artifacts(client, report_scope, 4) + assert len((await query(client, summary_scope, "ticket-summary"))["results"]) == 4 + + +def test_worker_timeout_is_local_to_its_plugin_and_code_update_recovers(tmp_path: Path) -> None: + asyncio.run(_recover_timed_out_plugin(tmp_path)) + + +async def _recover_timed_out_plugin(tmp_path: Path) -> None: + summary_registration = "/v1/admin/artifact-plugins/ticket-summaries" + async with running_server(tmp_path) as client: + report_scope = await create_scope(client, "timed-out-reports") + summary_scope = await create_scope(client, "healthy-summaries") + blocked = deploy_plugin( + EXAMPLE, + tmp_path / "blocked-v1", + [report_scope], + processing={"interval_seconds": 1, "max_workers": 1, "timeout_seconds": 5}, + ) + entry = blocked / "plugin.py" + entry.write_text( + entry + .read_text() + .replace("import hashlib", "import asyncio\nimport hashlib") + .replace(" drafts = []", " await asyncio.Future()\n drafts = []") + ) + healthy = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "healthy-v1", [summary_scope]) + for registration, directory in ((REGISTRATION, blocked), (summary_registration, healthy)): + await reload_plugin(client, tmp_path, directory, 0, registration=registration) + assert all(item["outcome"] == "accepted" for item in (await enable_all(client)).values()) + await plugin_state(client, "enabled") + await plugin_state(client, "enabled", summary_registration) + await capture(client, report_scope, "timed-out-ticket") + await capture(client, summary_scope, "healthy-ticket") + await wait_for_worker_metric(client, "timeouts_total") + assert await artifacts(client, report_scope, 0) == [] + healthy_result = await contents(client, summary_scope, "ticket-summary", 1) + assert healthy_result[0]["content"]["source_ids"] == ["healthy-ticket"] + assert (await plugin_state(client, "enabled", summary_registration))["binding_generation"] == 2 + recovered = deploy_plugin( + EXAMPLE, + tmp_path / "report-v2", + [report_scope], + version="2", + config={"title_prefix": "Recovered"}, + ) + await reload_plugin(client, tmp_path, recovered, 2, "enabled") + result = await contents(client, report_scope, "incident-report", 1) + assert result[0]["revision"] == 1 + assert result[0]["content"]["title"] == "Recovered: timed-out-ticket" + assert len((await query(client, summary_scope, "ticket-summary"))["results"]) == 1 + + +def test_enable_all_fixes_its_snapshot_and_reports_concurrent_changes(tmp_path: Path) -> None: + asyncio.run(_enable_all_with_concurrent_changes(tmp_path)) + + +async def _enable_all_with_concurrent_changes(tmp_path: Path) -> None: + async with running_server(tmp_path) as client: + scope = await create_scope(client, "batch-management") + report = deploy_plugin(EXAMPLE, tmp_path / "report-v1", [scope]) + gate = tmp_path / "validation-paused" + entered = tmp_path / "validation-entered" + entry = report / "plugin.py" + # A user module with a slow import allows independent HTTP management during validation. + entry.write_text( + entry.read_text() + + f""" + +from pathlib import Path +import time +if Path({str(gate)!r}).exists(): + Path({str(entered)!r}).write_text("entered") + while Path({str(gate)!r}).exists(): + time.sleep(0.05) +""" + ) + await reload_plugin(client, tmp_path, report, 0) + summary = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "summary-v1", [scope]) + publish_plugin(tmp_path, summary) + # Merely publishing an entry after startup does not make it eligible for a batch enable. + assert (await client.get(SUMMARY_REGISTRATION)).status_code == 404 + gate.touch() + batch = asyncio.create_task(enable_all(client, expected_failures=frozenset({"incident-reports"}))) + try: + async with asyncio.timeout(10): + while True: + if entered.exists(): + break + await asyncio.sleep(0.05) + await reload_plugin(client, tmp_path, summary, 0, registration=SUMMARY_REGISTRATION) + replacement = deploy_plugin(EXAMPLE, tmp_path / "report-v2", [scope], version="2") + updated = await reload_plugin(client, tmp_path, replacement, 1) + assert updated["binding_generation"] == 2 + finally: + gate.unlink(missing_ok=True) + results = await batch + assert set(results) == {"incident-reports"}, "A batch must not include plugins loaded after its snapshot" + assert results["incident-reports"]["outcome"] == "failed" + assert results["incident-reports"]["error"] == "conflict" + assert results["incident-reports"]["state"]["binding_generation"] == 2 + assert (await plugin_state(client, "disabled"))["binding_generation"] == 2 + assert (await plugin_state(client, "disabled", SUMMARY_REGISTRATION))["binding_generation"] == 1 + assert all(item["outcome"] == "accepted" for item in (await enable_all(client)).values()) + # Transitions and completed enables are both ineligible for an immediate repeat. + assert all(item["outcome"] == "skipped" for item in (await enable_all(client)).values()) + await plugin_state(client, "enabled") + await plugin_state(client, "enabled", SUMMARY_REGISTRATION) + + +def test_failed_restore_waits_for_explicit_reload_and_preserves_progress(tmp_path: Path) -> None: + asyncio.run(_repair_failed_restore(tmp_path)) + + +async def _repair_failed_restore(tmp_path: Path) -> None: + async with running_server(tmp_path) as client: + scope = await create_scope(client, "failed-restore") + healthy_scope = await create_scope(client, "healthy") + report = deploy_plugin(EXAMPLE, tmp_path / "report-v1", [scope]) + summary = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "summary-v1", [healthy_scope]) + await reload_plugin(client, tmp_path, report, 0) + await change(client, "enable", 1, "enabled") + await reload_plugin(client, tmp_path, summary, 0, registration=SUMMARY_REGISTRATION) + await capture(client, scope, "before-failure") + await artifacts(client, scope, 1) + # An unavailable saved version must fail restoration even if the discovery entry points at a valid update. + report.rename(tmp_path / "report-v1-unavailable") + replacement = deploy_plugin( + EXAMPLE, tmp_path / "report-v2", [scope], version="2", config={"title_prefix": "Repaired"} + ) + publish_plugin(tmp_path, replacement) + async with running_server(tmp_path) as client: + failed = await plugin_state(client, "failed") + assert failed["binding_generation"] == 3 + assert failed["last_error"] + await capture(client, scope, "during-failure") + results = await enable_all(client) + assert results["incident-reports"]["outcome"] == "skipped" + assert results["ticket-summaries"]["outcome"] == "accepted" + await plugin_state(client, "enabled", SUMMARY_REGISTRATION) + await capture(client, healthy_scope, "healthy-ticket") + await artifacts(client, healthy_scope, 1, "ticket-summary") + assert len((await query(client, scope))["results"]) == 1 + # Failed is durable, never auto-enabled by startup or by enable-all. + async with running_server(tmp_path) as client: + state = await plugin_state(client, "failed") + assert state["binding_generation"] == failed["binding_generation"] + assert all(item["outcome"] == "skipped" for item in (await enable_all(client)).values()) + invalid = await client.post(f"{REGISTRATION}/enable", json={"expected_generation": state["binding_generation"]}) + assert invalid.status_code == 422, invalid.text + state = await reload_plugin(client, tmp_path, replacement, state["binding_generation"]) + assert await artifacts(client, scope, 1) + assert (await enable_all(client))["incident-reports"]["outcome"] == "accepted" + await plugin_state(client, "enabled") + results = await contents(client, scope, "incident-report", 2) + repaired = next(item for item in results if item["sources"][0]["source_id"] == "during-failure") + assert repaired["revision"] == 1 + assert repaired["content"]["title"] == "Repaired: during-failure" + assert len((await query(client, healthy_scope, "ticket-summary"))["results"]) == 1 diff --git a/tests/e2e/test_artifact_vector_http.py b/tests/e2e/test_artifact_vector_http.py new file mode 100644 index 0000000000..285c46f829 --- /dev/null +++ b/tests/e2e/test_artifact_vector_http.py @@ -0,0 +1,128 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Installed Server and spawned generation Worker share text/vector/hybrid search.""" + +import asyncio + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from tests.builtin.persistence.test_artifact_indexes import PROFILE, body +from tests.e2e.test_artifact_plugin_search import _PIPELINE, _write_report +from tests.e2e.test_artifact_plugins import ( + EXAMPLE, + create_scope, + deploy_plugin, + enable_all, + reload_plugin, + running_server, +) + +_DECLARATION = """ +from powercontext.artifact_plugins import ArtifactSearchDeclaration, TextIndex, VectorIndex, FragmentProjection +from powercontext.builtin.artifacts.memory import EmbeddingProfile + +class Feature(BaseModel): + id: str + vector: list[float] + profile: EmbeddingProfile + +class SearchReport(IncidentReport): + private_text: str + fragments: list[Feature] + +families = (ArtifactFamily( + family="incident-report", schema_version="1", content_model=SearchReport, + search=ArtifactSearchDeclaration( + version="1", modes=("text", "vector", "hybrid"), default_mode="text", + indexes=( + TextIndex(name="lexical", fields=("title", "evidence_text"), projection_version="1"), + VectorIndex(name="visual", projection_version="1", source="content", + profile=EmbeddingProfile(profile_id="features-v1", model="fixture", dimension=2, normalization="none"), + vector_field="vector", profile_field="profile", + fragments=FragmentProjection(array_field="fragments", id_field="id")), + ), + ), +),) +inputs = InputContract(projections=inputs.projections, artifact_families=("experience", "skill", "profile")) +""" + + +def test_vector_search_through_http_and_spawned_worker(migration_backend): + config, directory = migration_backend.config, migration_backend.directory + database = config.model_dump(mode="json") + if isinstance(config, OceanBaseConfig): + database["url"] = config.url.get_secret_value() + + async def register(): + async with running_server(directory, database_settings=database) as client: + scope = await create_scope(client, "vector-search") + plugin = deploy_plugin(EXAMPLE, directory / "vector-plugin", [scope]) + entry = plugin / "plugin.py" + pipeline = _PIPELINE.replace( + " drafts = []", + """ # These use the built-in Searchers, not the custom-family declaration parser. + assert not (await self.ctx.search_artifacts("experience", "Outage")).matches + assert not (await self.ctx.search_artifacts("skill", "Outage")).matches + found = await self.ctx.search_artifacts("incident-report", "Report") + assert all("evidence_text" in artifact.content.model_dump() for artifact in found.artifacts) + for artifact in found.artifacts: + await self.ctx.read_revision(artifact.as_ref()) + drafts = []""", + ).replace( + "content=value, source_refs=(source.ref,),", + "content=value, source_refs=(source.ref,), artifact_refs=tuple(artifact.as_ref() for artifact in found.artifacts),", + ) + entry.write_text(entry.read_text() + _DECLARATION + pipeline, encoding="utf-8") + await reload_plugin(client, directory, plugin, 0) + return scope + + scope = asyncio.run(register()) + + async def exercise(): + async with running_server(directory, database_settings=database) as client: + await enable_all(client) + first = await _write_report( + client, scope, "first", "a", "数据库 timeout", fragments=body("", *([[0, 0]] * 8))["fragments"] + ) + second = await _write_report( + client, scope, "second", "b", "数据库 normal", fragments=body("", [1, 0])["fragments"] + ) + assert second["artifacts"] == [{"family": "incident-report", "artifact_id": "a", "revision": 1}] + endpoint = f"/v1/scopes/{scope}/artifacts/incident-report/search" + vector = {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]} + request = {"mode": "vector", "query_vector": vector, "limit": 2, "include_scores": True} + for mode in ("vector", "hybrid"): + response = await client.post( + endpoint, json={**request, "mode": mode, **({"query": "数据库"} if mode == "hybrid" else {})} + ) + assert response.status_code == 200, response.text + results = response.json()["results"] + assert [result["artifact_id"] for result in results] == ["a", "b"] + assert results[0]["content"] == first["content"] + assert set(results[0]) == {"family", "artifact_id", "revision", "content", "lineage", "scores"} + for invalid in ( + {"query": "Report", "indexes": {"vector": ["visual"]}}, + {"query": "Report", "indexes": {"text": []}}, + {**request, "query_vector": {**vector, "vector": [1]}}, + { + **request, + "mode": "hybrid", + "query": "Report", + "fusion": {"method": "rrf", "params": {"weights": {"text": 0, "vector": 0}}}, + }, + ): + response = await client.post(endpoint, json=invalid) + assert response.status_code == 422, response.text + unavailable = await client.post(endpoint, json={"mode": "vector", "query": "Report"}) + assert unavailable.status_code == 503 + denied = await client.post(endpoint, json=request, headers={"Authorization": ""}) + assert denied.status_code == 401 + await _write_report(client, scope, "update", "a", "recovered", revision=2, fragments=[]) + response = await client.post(endpoint, json=request) + assert [item["artifact_id"] for item in response.json()["results"]] == ["b"] + historical = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/a/revisions/1") + assert historical.json()["content"] == first["content"] + + asyncio.run(exercise()) diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py index 34c4f483dd..3cc9203f6e 100644 --- a/tests/test_api_contract.py +++ b/tests/test_api_contract.py @@ -327,6 +327,7 @@ def test_capabilities_report_semantics_without_runtime_tuning_values() -> None: assert set(properties) == { "artifact_dreaming", "artifact_dreaming_operations", + "custom_artifact_schemas", "source_types", "artifact_families", "memory_extraction", @@ -995,7 +996,8 @@ def test_standard_artifact_reads_share_topic_memory_family_and_display_metadata( for path in read_paths: family = next(parameter for parameter in paths[path]["get"]["parameters"] if parameter["name"] == "family") assert family["schema"] == {"$ref": "#/components/schemas/ArtifactReadFamily"} - assert "topic-memory" in contract["components"]["schemas"]["ArtifactReadFamily"]["enum"] + http_models.ArtifactReadFamily.model_validate("topic-memory") + http_models.ArtifactReadFamily.model_validate("incident-report") assert "topic-memory" in contract["components"]["schemas"]["BaseArtifactFamily"]["enum"] item = http_models.ArtifactCollectionItem.model_validate({ "scope_id": "scope-a", @@ -1010,7 +1012,7 @@ def test_standard_artifact_reads_share_topic_memory_family_and_display_metadata( "published_at": "2026-09-10T00:00:00Z", "source_count": 2, }) - assert item.family.value == "topic-memory" + assert item.family.root == "topic-memory" assert item.title == "A topic" assert item.source_count == 2 diff --git a/tests/test_artifact_index_maintenance.py b/tests/test_artifact_index_maintenance.py new file mode 100644 index 0000000000..30fe78af3e --- /dev/null +++ b/tests/test_artifact_index_maintenance.py @@ -0,0 +1,257 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Idle index maintenance must not require SQLite's writer lock or delay startup.""" + +import asyncio + +import pytest +from sqlalchemy import insert, select + +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchUnavailable, ArtifactSearchUnsupported +from powercontext.builtin.inference.models import EmbeddingResult +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ARTIFACT_SEARCH_STATES_TABLE, SCOPES_TABLE +from powercontext.server.artifact_plugins import ArtifactPluginManager +from tests.builtin.persistence.test_artifact_indexes import PROFILE, register, service, store, write + + +@pytest.mark.parametrize("after_rebuild", [False, True]) +def test_unchanged_maintenance_does_not_wait_for_an_unrelated_writer(tmp_path, after_rebuild): + async def run(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'store.db'}", busy_timeout_ms=50) + async with store(config) as (db, builder, publisher): + declaration = ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="text", fields=("text",), projection_version="1"),), + modes=("text",), + default_mode="text", + ) + await register(db, "reports", declaration) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库故障"}) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + if after_rebuild: + await builder.refresh("scope", "reports", rebuild=True) + assert await builder.build("scope", "reports") + # A separate connection holds SQLite's writer lock. Idle maintenance + # must still complete using WAL reads, without a busy timeout failure. + async with db.engine.connect() as blocker: + await blocker.exec_driver_sql("BEGIN IMMEDIATE") + try: + assert await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + finally: + await blocker.rollback() + assert (await service(db, "reports").search("scope", "reports", {"query": "故障"})).matches + + asyncio.run(run()) + + +def test_manager_startup_does_not_wait_for_existing_data_encoding(): + async def run(): + started, release = asyncio.Event(), asyncio.Event() + + class Encoder: + profile = PROFILE + + async def embed(self, texts): + started.set() + await release.wait() + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + async with store() as (db, _, publisher): + await register( + db, + "startup_reports", + ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", fields=("text",), projection_version="1", source="text", profile=PROFILE + ), + ), + modes=("vector",), + default_mode="vector", + ), + ) + async with db.transaction() as connection: + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id="scope", + title="", + summary="", + scope_id_search="scope", + title_search="", + summary_search="", + version=1, + ) + ) + await write(db, publisher, "scope", "startup_reports", "a", {"text": "数据库故障"}) + manager = ArtifactPluginManager( + database=db, + config=SQLiteConfig(), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + embedding_model=Encoder(), + ) + try: + async with asyncio.timeout(5): + await manager.__aenter__() + await started.wait() + release.set() + async with asyncio.timeout(5): + while True: + try: + result = await service(db, "startup_reports").search( + "scope", + "startup_reports", + {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1.0, 0.0]}}, + ) + assert result.matches + break + except ArtifactSearchUnavailable: + await asyncio.sleep(0.01) + finally: + release.set() + await manager.__aexit__() + + asyncio.run(run()) + + +def test_manager_discovers_first_write_without_creating_empty_scope_states(): + async def run(): + async with store() as (db, _, publisher): + await register( + db, + "reports", + ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="text", fields=("text",), projection_version="1"),), + modes=("text",), + default_mode="text", + ), + ) + async with db.transaction() as connection: + for scope in ("active", "empty"): + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id=scope, + title="", + summary="", + scope_id_search=scope, + title_search="", + summary_search="", + version=1, + ) + ) + async with ArtifactPluginManager( + database=db, + config=SQLiteConfig(), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + ): + # Allow startup discovery to finish before the first content arrives. + await asyncio.sleep(0.1) + assert not (await service(db, "reports").search("empty", "reports", {"query": "故障"})).matches + with pytest.raises(ArtifactSearchUnsupported): + await service(db, "reports").search( + "empty", "reports", {"query": "故障", "indexes": {"text": ["unknown"]}} + ) + await write(db, publisher, "active", "reports", "a", {"text": "数据库故障"}) + async with asyncio.timeout(6): + while True: + try: + result = await service(db, "reports").search("active", "reports", {"query": "故障"}) + assert result.matches + break + except ArtifactSearchUnavailable: + await asyncio.sleep(0.05) + async with db.transaction() as connection: + scopes = (await connection.execute(select(ARTIFACT_SEARCH_STATES_TABLE.c.scope_id))).scalars().all() + assert scopes == ["active"] + + asyncio.run(run()) + + +def test_manager_recovers_failed_build_and_cleans_before_rebuilding(): + async def run(): + from powercontext.builtin.persistence.artifact_indexes import COVERAGE, STATES + + failed = asyncio.Event() + + class Encoder: + profile = PROFILE + available = False + + async def embed(self, texts): + if not self.available: + failed.set() + raise RuntimeError("encoder unavailable") # noqa: TRY003 + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + encoder = Encoder() + async with store(model=encoder) as (db, builder, publisher): + await register( + db, + "reports", + ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", fields=("text",), projection_version="1", source="text", profile=PROFILE + ), + ), + modes=("vector",), + default_mode="vector", + ), + ) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库故障"}) + + def manager(): + return ArtifactPluginManager( + database=db, + config=SQLiteConfig(), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + embedding_model=encoder, + ) + + async with manager(): + async with asyncio.timeout(5): + await failed.wait() + encoder.available = True + async with manager(): + async with asyncio.timeout(6): + while True: + try: + result = await service(db, "reports").search( + "scope", + "reports", + {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}}, + ) + assert result.matches + break + except ArtifactSearchUnavailable: + await asyncio.sleep(0.05) + async with db.transaction() as connection: + old = await connection.scalar(select(STATES.c.build_id).where(STATES.c.state == "ready")) + await builder.refresh("scope", "reports", rebuild=True) + async with asyncio.timeout(6): + while True: + async with db.transaction() as connection: + state = await connection.scalar(select(STATES.c.state).where(STATES.c.build_id != old)) + remains = await connection.scalar( + select(COVERAGE.c.artifact_id).where(COVERAGE.c.build_id == old) + ) + if state == "ready" and remains is None: + break + await asyncio.sleep(0.05) + + asyncio.run(run()) diff --git a/tests/test_artifact_plugin_validation.py b/tests/test_artifact_plugin_validation.py new file mode 100644 index 0000000000..709b7b55f0 --- /dev/null +++ b/tests/test_artifact_plugin_validation.py @@ -0,0 +1,81 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +"""Declaration validation in a real child within its existing startup budget.""" + +import asyncio +import os + +import httpx +import pytest + +from powercontext.builtin.artifact_plugin_validation import validate_plugin +from tests.e2e.test_artifact_plugins import EXAMPLE, REGISTRATION, deploy_plugin, enable_all + + +def test_declaration_validation_does_not_spend_its_budget_starting_the_runtime(tmp_path, monkeypatch): + plugin = deploy_plugin(EXAMPLE, tmp_path / "plugin", None, config={}) + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text() + + '\nimport os\nos.environ["PC_TEST_PLUGIN_IMPORTED"] = "child"\n' + + 'print("plugin import output is not the declaration protocol")\n' + ) + monkeypatch.delenv("PC_TEST_PLUGIN_IMPORTED", raising=False) + # A slow runtime bootstrap must not consume declaration validation's 15-second budget. + # This hook is installed only in new Python processes, not in the pytest process. + hook = tmp_path / "startup" + hook.mkdir() + (hook / "sitecustomize.py").write_text( + "import sys, time\n" + "class SlowRuntime:\n" + " def find_spec(self, fullname, path=None, target=None):\n" + " if fullname == 'powercontext.builtin.runtime':\n" + " time.sleep(20)\n" + "sys.meta_path.insert(0, SlowRuntime())\n" + ) + monkeypatch.setenv("PYTHONPATH", os.pathsep.join(filter(None, (str(hook), os.environ.get("PYTHONPATH"))))) + + async def run(): + described = await validate_plugin(str(plugin)) + restored = await validate_plugin(str(plugin), described["manifest"]) + assert "PC_TEST_PLUGIN_IMPORTED" not in os.environ + assert restored == described + assert described["declaration"]["family"] == "incident-report" + assert described["manifest"]["config"]["title_prefix"] == "Incident report" + + asyncio.run(run()) + + +def test_invalid_declaration_is_rejected_by_the_validation_child(tmp_path): + plugin = deploy_plugin(EXAMPLE, tmp_path / "invalid", None, config={"unknown": True}) + + async def run(): + with pytest.raises(ValueError, match="plugin declaration or configuration validation failed"): + await validate_plugin(str(plugin)) + + asyncio.run(run()) + + +def test_enable_all_reports_unexpected_item_failure_immediately(): + item = { + "binding_name": "incident-reports", + "outcome": "failed", + "status_url": REGISTRATION, + "state": {"phase": "disabled"}, + "error": "invalid_plugin", + } + + async def run(): + transport = httpx.MockTransport(lambda request: httpx.Response(200, json={"items": [item]})) + async with httpx.AsyncClient(base_url="http://test", transport=transport) as client: + with pytest.raises(AssertionError, match="invalid_plugin"): + await enable_all(client) + results = await enable_all(client, expected_failures=frozenset({"incident-reports"})) + assert results["incident-reports"] == item + + asyncio.run(run()) From d6820d884ccd240712df477e459622f60fd6f5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Mon, 26 Oct 2026 17:10:42 +0800 Subject: [PATCH 02/10] fix(artifacts): validate unready writes and preserve plugin leases --- .../builtin/persistence/artifact_indexes.py | 66 +++++++++++------ src/powercontext/server/artifact_plugins.py | 3 +- .../persistence/test_artifact_indexes.py | 74 +++++++++++++++++++ tests/test_artifact_plugin_leases.py | 54 ++++++++++++++ 4 files changed, 174 insertions(+), 23 deletions(-) create mode 100644 tests/test_artifact_plugin_leases.py diff --git a/src/powercontext/builtin/persistence/artifact_indexes.py b/src/powercontext/builtin/persistence/artifact_indexes.py index f2d81bf163..86d35f959f 100644 --- a/src/powercontext/builtin/persistence/artifact_indexes.py +++ b/src/powercontext/builtin/persistence/artifact_indexes.py @@ -329,6 +329,34 @@ def vector_bytes(values, profile): return encoded +def index_parts(index, content): + if isinstance(index, VectorIndex) and index.fragments is not None: + parts = content[index.fragments.array_field] + if not isinstance(parts, list): + raise ValueError("fragment projection requires a persistent array") # noqa: TRY003 + else: + parts = [content] + seen = set() + for offset, part in enumerate(parts): + fragment_id = "__whole__" + locator = {"fields": list(index.fields)} + if isinstance(index, VectorIndex) and index.fragments is not None: + fragment_id = part[index.fragments.id_field] + if not isinstance(fragment_id, str) or not 1 <= len(fragment_id) <= 128 or fragment_id == "__whole__": + raise ValueError("fragment identity must be a bounded persistent string") # noqa: TRY003 + locator = {"array": index.fragments.array_field, "id": fragment_id, "offset": offset} + if fragment_id in seen: + raise ValueError("fragment identities must be unique") # noqa: TRY003 + seen.add(fragment_id) + yield part, fragment_id, locator + + +def content_vector(index, part): + if EmbeddingProfile.model_validate(part[index.profile_field]) != index.profile: + raise ValueError("persistent vector profile differs from the declared index") # noqa: TRY003 + return vector_bytes(part[index.vector_field], index.profile) + + @dataclass(frozen=True) class PreparedIndex: definition: dict[str, Any] @@ -350,28 +378,11 @@ def __init__(self, database, embedding_model=None): self.database = database self.embedding_model = embedding_model - async def prepare_index(self, definition, content): # noqa: C901 - Keep the complete projection or publication invariant together. + async def prepare_index(self, definition, content): index = parse_index(definition) - if isinstance(index, VectorIndex) and index.fragments is not None: - parts = content[index.fragments.array_field] - if not isinstance(parts, list): - raise ValueError("fragment projection requires a persistent array") # noqa: TRY003 - else: - parts = [content] fragments = [] texts = [] - seen = set() - for offset, part in enumerate(parts): - fragment_id = "__whole__" - locator = {"fields": list(index.fields)} - if isinstance(index, VectorIndex) and index.fragments is not None: - fragment_id = part[index.fragments.id_field] - if not isinstance(fragment_id, str) or not 1 <= len(fragment_id) <= 128 or fragment_id == "__whole__": - raise ValueError("fragment identity must be a bounded persistent string") # noqa: TRY003 - locator = {"array": index.fragments.array_field, "id": fragment_id, "offset": offset} - if fragment_id in seen: - raise ValueError("fragment identities must be unique") # noqa: TRY003 - seen.add(fragment_id) + for part, fragment_id, locator in index_parts(index, content): value = {"fragment_id": fragment_id, "fragment_locator": identity_json(locator).decode()} if isinstance(index, TextIndex) or index.source == "text": fields = [part[field] for field in index.fields] @@ -383,9 +394,7 @@ async def prepare_index(self, definition, content): # noqa: C901 - Keep the com else: texts.append(original) else: - if EmbeddingProfile.model_validate(part[index.profile_field]) != index.profile: - raise ValueError("persistent vector profile differs from the declared index") # noqa: TRY003 - value["embedding"] = vector_bytes(part[index.vector_field], index.profile) + value["embedding"] = content_vector(index, part) value["input_hash"] = digest({"projection": definition["definition_fingerprint"], "part": part}) fragments.append(value) if texts: @@ -440,6 +449,19 @@ async def prepare(self, scope, family, schema_version, content): async def validate_prepared(self, connection, scope, family, schema_version, content, prepared): state = await lock_state(connection, scope, family) if state["state"] != "ready": + durable = await connection.scalar( + select(SCHEMAS.c.declaration).where( + SCHEMAS.c.family == family, SCHEMAS.c.schema_version == schema_version + ) + ) + declaration = None if durable is None else search_declaration(durable) + if declaration is not None: + # Resource-independent content invariants hold even while projections are suspended. + for index in declaration.indexes: + if isinstance(index, VectorIndex): + for part, _, _ in index_parts(index, content): + if index.source == "content": + content_vector(index, part) return state expected = {entry["index_def_id"] for schema, entry in entries(state["manifest"]) if schema == schema_version} if prepared is None: diff --git a/src/powercontext/server/artifact_plugins.py b/src/powercontext/server/artifact_plugins.py index bd453334df..1f1b8513e2 100644 --- a/src/powercontext/server/artifact_plugins.py +++ b/src/powercontext/server/artifact_plugins.py @@ -25,6 +25,7 @@ from powercontext.builtin.dream.bindings import DREAM_BINDINGS from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.sources import SOURCE_PROCESSING_BINDINGS from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ( @@ -150,7 +151,7 @@ async def _ensure_supervisor(self) -> ArtifactProcessingSupervisors: ArtifactProcessingSupervisor( database=self.database, bindings=(), - lease_mode="single-process", + lease_mode="oceanbase" if isinstance(self.config, OceanBaseConfig) else "single-process", ) ) self.supervisor = ArtifactProcessingSupervisors((controller,)) diff --git a/tests/builtin/persistence/test_artifact_indexes.py b/tests/builtin/persistence/test_artifact_indexes.py index f3856d4b45..f4c80659ad 100644 --- a/tests/builtin/persistence/test_artifact_indexes.py +++ b/tests/builtin/persistence/test_artifact_indexes.py @@ -353,6 +353,80 @@ async def run(): asyncio.run(run()) +def test_initial_content_vector_validation_rejects_bad_record_without_requiring_index_resources(): + async def run(): + async with store() as (db, _, publisher): + base = declaration() + search_config = ArtifactSearchDeclaration( + version="1", + indexes=( + *base.indexes, + VectorIndex( + name="semantic", fields=("text",), source="text", profile=PROFILE, projection_version="1" + ), + ), + modes=base.modes, + default_mode="text", + defaults={ + "text": {"text": ("lexical",)}, + "vector": {"vector": ("visual",)}, + "hybrid": {"text": ("lexical",), "vector": ("visual",)}, + }, + ) + await register(db, "reports", search_config) + wrong = PROFILE.model_copy(update={"profile_id": "features-v2", "model": "second"}) + with pytest.raises(ValueError, match="profile differs"): + await write(db, publisher, "scope", "reports", "a", body("invalid", [1, 0], profile=wrong)) + async with db.transaction() as connection: + with pytest.raises(RepositoryNotFoundError): + await custom_artifact_repository("reports").latest(connection, "scope", "reports", "a") + + # The family is still unready: neither the text encoder nor physical vector spaces are needed for this write. + stored = await write(db, publisher, "scope", "reports", "a", body("valid", [1, 0])) + assert stored.revision == 1 + + asyncio.run(run()) + + +def test_repair_content_vector_validation_rejects_bad_revision_across_ready_transition(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + original = await write(db, publisher, "scope", "reports", "a", body("original", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + await builder.refresh("scope", "reports", rebuild=True) + + with pytest.raises(ValueError, match="dimension"): + await write(db, publisher, "scope", "reports", "a", body("invalid dimension", [1])) + async with db.transaction() as connection: + latest = await custom_artifact_repository("reports").latest(connection, "scope", "reports", "a") + assert latest.revision == original.revision + assert latest.content == original.content + + duplicate = body("duplicate fragments", [1, 0], [0, 1]) + duplicate["fragments"][1]["id"] = duplicate["fragments"][0]["id"] + with pytest.raises(ValueError, match="identities must be unique"): + await write(db, publisher, "scope", "reports", "a", duplicate) + + changed = body("changed while repairing", [0, 1]) + prepared = await publisher.prepare("scope", "reports", "1", changed) + assert await builder.build("scope", "reports") + repository = custom_artifact_repository("reports", prepared={"a": prepared}) + draft = RepositoryArtifactDraft( + family="reports", content=StoredArtifactContent(schema_version="1", content=changed) + ) + with pytest.raises(ValueError): + async with db.transaction() as connection: + await repository.revise(connection, "scope", original, draft) + async with db.transaction() as connection: + latest = await repository.latest(connection, "scope", "reports", "a") + assert latest.revision == original.revision + assert latest.content == original.content + + asyncio.run(run()) + + def test_vector_profile_is_validated_against_requested_index(): async def run(): async with store() as (db, builder, _): diff --git a/tests/test_artifact_plugin_leases.py b/tests/test_artifact_plugin_leases.py new file mode 100644 index 0000000000..1bcdae9e1e --- /dev/null +++ b/tests/test_artifact_plugin_leases.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026 OceanBase. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + +"""Plugin activation preserves the renewable Supervisor ownership contract.""" + +import asyncio + +from pydantic import SecretStr + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.server.artifact_plugins import ArtifactPluginManager +from tests.e2e.test_artifact_plugins import EXAMPLE, deploy_plugin + + +def test_first_plugin_activation_preserves_an_unexpired_oceanbase_owner(tmp_path): + deploy_plugin(EXAMPLE, tmp_path / "plugins" / "incident-reports", None) + + async def run(): + # SQLite hosts real lease state; OceanBaseConfig exercises configuration + # selection without claiming acceptance against an OceanBase server. + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as storage: + database = storage.database + leases = ArtifactProcessingLeaseRepository() + async with database.transaction() as connection: + owner = await leases.try_acquire(connection, "existing-background-owner", 3600) + assert owner is not None and owner.lease_expires_at is not None + fence = owner.fence("oceanbase") + async with ArtifactPluginManager( + database=database, + config=OceanBaseConfig(url=SecretStr("mysql+aoceanbase://test@localhost:2881/test?charset=utf8mb4")), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + plugins_dir=tmp_path / "plugins", + ) as manager: + registration = await manager.get("incident-reports") + assert registration.phase == "disabled" + await manager.change("incident-reports", registration.binding_generation, "enabled") + async with asyncio.timeout(10): + while (state := await manager.get("incident-reports")).phase == "activating": # noqa: ASYNC110 + await asyncio.sleep(0.01) + assert state.phase == "enabled", state.last_error + async with database.transaction() as connection: + assert await leases.require_fence(connection, fence) == owner + # Closing the standby must not release another Supervisor's term. + async with database.transaction() as connection: + assert await leases.require_fence(connection, fence) == owner + + asyncio.run(run()) From 64b5f8e426019c4bd242a49818eb52feec9d32b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Mon, 26 Oct 2026 18:02:25 +0800 Subject: [PATCH 03/10] fix(artifacts): resolve framework integration CI regressions --- desktop/VALIDATION.md | 8 ++ .../src/connections/compatibility.json | 19 +++++ e2e/bub/tests/test_access_audit.py | 7 ++ .../powercontext-tool-contract/CLAUDE.md | 4 +- .../incident_report/plugin.py | 11 ++- .../incident_report/plugin.yml | 14 ++++ .../artifact_plugins/ticket_summary/plugin.py | 11 ++- .../ticket_summary/plugin.yml | 14 ++++ .../plugin/powercontext_dify/contract.json | 24 +++--- .../dify/plugin/tools/pc_memory_get.yaml | 22 +++--- .../dify/plugin/tools/pc_memory_revise.yaml | 22 +++--- .../dsh/plugins/powercontext/lib/index.js | 77 +++++++++++++++++++ scripts/generate_api.py | 10 +++ src/powercontext/artifact_plugins.py | 11 ++- src/powercontext/artifacts/indexes.py | 11 ++- .../builtin/artifact_plugin_validation.py | 7 ++ .../builtin/persistence/artifact_indexes.py | 13 +++- .../persistence/artifact_search_schema.py | 11 ++- .../persistence/artifact_vector_space.py | 11 ++- .../builtin/persistence/custom_artifacts.py | 11 ++- .../persistence/processing_migration.py | 3 + .../builtin/persistence/sqlite/profile.py | 11 ++- .../builtin/runtime/artifact_plugins.py | 11 ++- .../builtin/runtime/artifact_reads.py | 11 ++- .../builtin/runtime/composition.py | 2 +- .../builtin/runtime/custom_artifact_search.py | 18 ++++- src/powercontext/http/_generated/models.py | 4 +- src/powercontext/server/app.py | 4 +- src/powercontext/server/artifact_plugins.py | 11 ++- .../persistence/test_artifact_indexes.py | 46 ++++++++++- .../test_custom_artifact_search.py | 11 ++- .../runtime/test_artifact_read_context.py | 11 ++- tests/conftest.py | 13 ++++ tests/e2e/test_artifact_index_backends.py | 11 ++- tests/e2e/test_artifact_plugin_search.py | 11 ++- tests/e2e/test_artifact_plugins.py | 11 ++- tests/e2e/test_artifact_vector_http.py | 11 ++- tests/test_api_contract.py | 21 ++++- tests/test_artifact_index_maintenance.py | 11 ++- tests/test_artifact_plugin_leases.py | 11 ++- tests/test_artifact_plugin_validation.py | 7 ++ 41 files changed, 505 insertions(+), 63 deletions(-) diff --git a/desktop/VALIDATION.md b/desktop/VALIDATION.md index 94c5a30e0c..6099898d45 100644 --- a/desktop/VALIDATION.md +++ b/desktop/VALIDATION.md @@ -475,3 +475,11 @@ and checks digest rejection and allowlisted projection; it does not qualify a re Use a disposable Server and Scope for installed save/search/read/copy checks. Unit tests do not prove those installed workflows. Validate actual Chinese IME, accessibility, Gatekeeper, signing and notarization independently before distributing outside the internal preview. + +## Server d6820d88 protocol qualification + +`sqlite-artifact-d6820d88-v1` identifies Server sources at `d6820d884ccd240712df477e459622f60fd6f5f5`, contract digest `f571a95d193f0dae91694ffcd6b37c93603a5ae731fddefbcdbfa9a5914762ae`, and wheel digest `5bcedfde3efa4ae028318a9b8848dec44b5604ac6f636f32bacb637ca5729f52`. The wheel `powercontext-0.1.dev0+gd6820d88-py3-none-any.whl` was built from a clean Git archive of that commit with an explicit archive-build version; all 563 runtime files match the archived sources byte for byte. + +On 2026-10-26, the native `ConnectionManager` and HTTP adapter passed all four built-wheel SQLite fixture modes: anonymous loopback, Bearer, HTTPS with an explicit CA/base path, and injected-provider/enforced access. The unchanged `server_probe.rs` assertions cover writes, search, exact historical reads, lost-response writes without replay, identity changes, revocation, and Scope pagination. The qualification records the same ten Desktop operations as the preceding profile; it does not qualify the new plugin administration operations. + +Validation used Linux, Python 3.12.12 and Rust 1.95.0. The unchanged connection, credential and transport modules and probe were built in an isolated native crate without Tauri/UI dependencies. On the loaded host, importing the Server took about 60 seconds, so the local fixture preparation allowance was 180 seconds instead of 20; protocol assertions and product timeouts were unchanged. This establishes native SQLite protocol compatibility, not full Tauri, installed-package, Windows/macOS service, vault or UI acceptance. diff --git a/desktop/src-tauri/src/connections/compatibility.json b/desktop/src-tauri/src/connections/compatibility.json index 0ccf064f97..dd60729eb9 100644 --- a/desktop/src-tauri/src/connections/compatibility.json +++ b/desktop/src-tauri/src/connections/compatibility.json @@ -189,5 +189,24 @@ "search_memory" ], "evidence": "desktop/VALIDATION.md#server-98275dbb-protocol-qualification" + }, + { + "id": "sqlite-artifact-d6820d88-v1", + "serverCommit": "d6820d884ccd240712df477e459622f60fd6f5f5", + "contractSha256": "f571a95d193f0dae91694ffcd6b37c93603a5ae731fddefbcdbfa9a5914762ae", + "artifactSha256": "5bcedfde3efa4ae028318a9b8848dec44b5604ac6f636f32bacb637ca5729f52", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#server-d6820d88-protocol-qualification" } ] diff --git a/e2e/bub/tests/test_access_audit.py b/e2e/bub/tests/test_access_audit.py index d438f94268..5c7fa9db4a 100644 --- a/e2e/bub/tests/test_access_audit.py +++ b/e2e/bub/tests/test_access_audit.py @@ -260,6 +260,13 @@ def test_a_handoff_report_over_every_scope_reaches_them_although_it_is_authorize "set_scope_binding", "clear_scope_binding", "register_source_definition", + "list_artifact_plugins", + "get_artifact_plugin", + "enable_all_artifact_plugins", + "reload_artifact_plugin", + "enable_artifact_plugin", + "disable_artifact_plugin", + "unload_artifact_plugin", "scan_external_skills", "list_external_skills", "resolve_external_skill", diff --git a/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md b/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md index 6abdcb4edb..930c1161d9 100644 --- a/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md +++ b/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md @@ -87,12 +87,12 @@ not replace those schemas or enforce their constraints. It contains no scenario ## mcp__powercontext__get_artifact -- Required: `artifact_id: string`, `family: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"`, `scope_id: string`. +- Required: `artifact_id: string`, `family: ArtifactReadFamily`, `scope_id: string`. - Optional: `If-None-Match: string`. ## mcp__powercontext__get_artifact_revision -- Required: `artifact_id: string`, `family: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"`, `revision: integer`, `scope_id: string`. +- Required: `artifact_id: string`, `family: ArtifactReadFamily`, `revision: integer`, `scope_id: string`. - Optional: (none). ## mcp__powercontext__get_atomic_memory_state diff --git a/examples/artifact_plugins/incident_report/plugin.py b/examples/artifact_plugins/incident_report/plugin.py index c59aa6b608..d612266039 100644 --- a/examples/artifact_plugins/incident_report/plugin.py +++ b/examples/artifact_plugins/incident_report/plugin.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Generate one deterministic report for each exact ticket snapshot.""" diff --git a/examples/artifact_plugins/incident_report/plugin.yml b/examples/artifact_plugins/incident_report/plugin.yml index 81ebf7cac3..68313a4b35 100644 --- a/examples/artifact_plugins/incident_report/plugin.yml +++ b/examples/artifact_plugins/incident_report/plugin.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + id: example-incident-report version: "1" entry: ./plugin.py:create_pipeline diff --git a/examples/artifact_plugins/ticket_summary/plugin.py b/examples/artifact_plugins/ticket_summary/plugin.py index bec79ba338..cbb66d54fc 100644 --- a/examples/artifact_plugins/ticket_summary/plugin.py +++ b/examples/artifact_plugins/ticket_summary/plugin.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Summarize each Source window without an external model dependency.""" diff --git a/examples/artifact_plugins/ticket_summary/plugin.yml b/examples/artifact_plugins/ticket_summary/plugin.yml index dbdecc1ecc..3bf122fc71 100644 --- a/examples/artifact_plugins/ticket_summary/plugin.yml +++ b/examples/artifact_plugins/ticket_summary/plugin.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + id: example-ticket-summary version: "1" entry: ./plugin.py:create_pipeline diff --git a/integrations/dify/plugin/powercontext_dify/contract.json b/integrations/dify/plugin/powercontext_dify/contract.json index 662443e44d..03f169cc11 100644 --- a/integrations/dify/plugin/powercontext_dify/contract.json +++ b/integrations/dify/plugin/powercontext_dify/contract.json @@ -911,16 +911,9 @@ }, "ArtifactReadFamily": { "type": "string", - "enum": [ - "memory", - "atomic-memory", - "experience", - "skill", - "handoff", - "profile", - "prompt", - "topic-memory" - ] + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z][a-z0-9-]*$" }, "ArtifactReference": { "type": "object", @@ -977,6 +970,17 @@ "maxLength": 128, "pattern": "^[\\x21-\\x7E]+$" }, + "schema_version": { + "anyOf": [ + { + "type": "string", + "description": "Persisted schema version for a custom Artifact family." + }, + { + "type": "null" + } + ] + }, "revision": { "type": "integer", "minimum": 1 diff --git a/integrations/dify/plugin/tools/pc_memory_get.yaml b/integrations/dify/plugin/tools/pc_memory_get.yaml index c02c011377..ffb439930a 100644 --- a/integrations/dify/plugin/tools/pc_memory_get.yaml +++ b/integrations/dify/plugin/tools/pc_memory_get.yaml @@ -77,18 +77,13 @@ output_schema: family: $ref: '#/definitions/Result/properties/family' type: string - enum: &id001 - - memory - - atomic-memory - - experience - - skill - - handoff - - profile - - prompt - - topic-memory artifact_id: $ref: '#/definitions/Result/properties/artifact_id' type: string + schema_version: + $ref: '#/definitions/Result/properties/schema_version' + type: string + description: Persisted schema version for a custom Artifact family. revision: $ref: '#/definitions/Result/properties/revision' type: number @@ -180,12 +175,19 @@ output_schema: pattern: .*\S.* family: type: string - enum: *id001 + minLength: 1 + maxLength: 128 + pattern: ^[a-z][a-z0-9-]*$ artifact_id: type: string minLength: 1 maxLength: 128 pattern: ^[\x21-\x7E]+$ + schema_version: + anyOf: + - type: string + description: Persisted schema version for a custom Artifact family. + - type: 'null' revision: type: integer minimum: 1 diff --git a/integrations/dify/plugin/tools/pc_memory_revise.yaml b/integrations/dify/plugin/tools/pc_memory_revise.yaml index 2353764748..178b0407f3 100644 --- a/integrations/dify/plugin/tools/pc_memory_revise.yaml +++ b/integrations/dify/plugin/tools/pc_memory_revise.yaml @@ -137,18 +137,13 @@ output_schema: family: $ref: '#/definitions/Result/properties/family' type: string - enum: &id001 - - memory - - atomic-memory - - experience - - skill - - handoff - - profile - - prompt - - topic-memory artifact_id: $ref: '#/definitions/Result/properties/artifact_id' type: string + schema_version: + $ref: '#/definitions/Result/properties/schema_version' + type: string + description: Persisted schema version for a custom Artifact family. revision: $ref: '#/definitions/Result/properties/revision' type: number @@ -229,12 +224,19 @@ output_schema: pattern: .*\S.* family: type: string - enum: *id001 + minLength: 1 + maxLength: 128 + pattern: ^[a-z][a-z0-9-]*$ artifact_id: type: string minLength: 1 maxLength: 128 pattern: ^[\x21-\x7E]+$ + schema_version: + anyOf: + - type: string + description: Persisted schema version for a custom Artifact family. + - type: 'null' revision: type: integer minimum: 1 diff --git a/integrations/dsh/plugins/powercontext/lib/index.js b/integrations/dsh/plugins/powercontext/lib/index.js index c1df7f1d68..6c1ffd3f96 100644 --- a/integrations/dsh/plugins/powercontext/lib/index.js +++ b/integrations/dsh/plugins/powercontext/lib/index.js @@ -315,6 +315,83 @@ const OPERATIONS = { successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { + method: "GET", + path: "/v1/admin/artifact-plugins", + location: null, + scopeMode: "none", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + get_artifact_plugin: { + method: "GET", + path: "/v1/admin/artifact-plugins/{binding_name}", + location: null, + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + enable_all_artifact_plugins: { + method: "POST", + path: "/v1/admin/artifact-plugins/enable-all", + location: null, + scopeMode: "none", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + reload_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/reload", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + enable_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/enable", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + disable_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/disable", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + unload_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/unload", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, get_capabilities: { method: "GET", path: "/v1/capabilities", diff --git a/scripts/generate_api.py b/scripts/generate_api.py index 06142b13e0..0ca464af33 100644 --- a/scripts/generate_api.py +++ b/scripts/generate_api.py @@ -127,6 +127,16 @@ def _generate_models( transport_contract.paths = {} if transport_contract.components is None or transport_contract.components.schemas is None: raise ContractGenerationError("components.schemas", None) + # Read-family values remain strings in the SDK, including custom family names. + # A referenced scalar otherwise becomes a RootModel and breaks identity comparisons. + family_schema = transport_contract.components.schemas["ArtifactReadFamily"] + if not isinstance(family_schema, Schema): + raise ContractGenerationError("ArtifactReadFamily schema", family_schema) # noqa: TRY003 + for name in ("ArtifactCollectionItem", "ArtifactRevision"): + schema = transport_contract.components.schemas[name] + if not isinstance(schema, Schema) or schema.properties is None: + raise ContractGenerationError("Artifact read schema", name) # noqa: TRY003 + schema.properties["family"] = family_schema.model_copy(deep=True) exclusive_schemas = { name: schema for name, schema in transport_contract.components.schemas.items() diff --git a/src/powercontext/artifact_plugins.py b/src/powercontext/artifact_plugins.py index e97634144a..ade7db50ea 100644 --- a/src/powercontext/artifact_plugins.py +++ b/src/powercontext/artifact_plugins.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Declarations and generation contracts for Worker-loaded Artifact plugins.""" diff --git a/src/powercontext/artifacts/indexes.py b/src/powercontext/artifacts/indexes.py index 2be73ac3d6..91ee498e2e 100644 --- a/src/powercontext/artifacts/indexes.py +++ b/src/powercontext/artifacts/indexes.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Declarative, persistent search projections for ordinary Artifact families.""" diff --git a/src/powercontext/builtin/artifact_plugin_validation.py b/src/powercontext/builtin/artifact_plugin_validation.py index dc5837f39c..cb0de0474e 100644 --- a/src/powercontext/builtin/artifact_plugin_validation.py +++ b/src/powercontext/builtin/artifact_plugin_validation.py @@ -1,9 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Isolated Artifact declaration validation without starting the generation runtime.""" diff --git a/src/powercontext/builtin/persistence/artifact_indexes.py b/src/powercontext/builtin/persistence/artifact_indexes.py index 86d35f959f..de620583bc 100644 --- a/src/powercontext/builtin/persistence/artifact_indexes.py +++ b/src/powercontext/builtin/persistence/artifact_indexes.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Transactional custom Artifact projections, build coordination and rebuilds.""" @@ -24,6 +33,7 @@ from powercontext.builtin.artifacts.memory.canonical import canonical_embedding, canonical_json from powercontext.builtin.artifacts.memory.models import EmbeddingProfile from powercontext.builtin.artifacts.search import TEXT_ANALYZER_VERSION, analyze_text +from powercontext.builtin.persistence.sqlite.profile import ensure_sqlite_vector_extension from powercontext.builtin.persistence.tables import ( ARTIFACT_FAMILY_SCHEMAS_TABLE as SCHEMAS, ) @@ -903,6 +913,7 @@ async def _coverage_valid(self, connection, scope, family, build_id, manifest): if isinstance(index, VectorIndex): probe = vector_bytes([1.0] * index.profile.dimension, index.profile) if connection.dialect.name == "sqlite": + await ensure_sqlite_vector_extension(connection) await connection.execute(text("SELECT vec_distance_l2(:vector, :vector)"), {"vector": probe}) else: from sqlalchemy import bindparam diff --git a/src/powercontext/builtin/persistence/artifact_search_schema.py b/src/powercontext/builtin/persistence/artifact_search_schema.py index 1ebf94bc86..b60e65568d 100644 --- a/src/powercontext/builtin/persistence/artifact_search_schema.py +++ b/src/powercontext/builtin/persistence/artifact_search_schema.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Shared search build state and exact, index-isolated projections.""" diff --git a/src/powercontext/builtin/persistence/artifact_vector_space.py b/src/powercontext/builtin/persistence/artifact_vector_space.py index 8d3dad7af8..b214a2e14d 100644 --- a/src/powercontext/builtin/persistence/artifact_vector_space.py +++ b/src/powercontext/builtin/persistence/artifact_vector_space.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Server-owned OceanBase/seekDB VECTOR spaces, provisioned during database initialization.""" diff --git a/src/powercontext/builtin/persistence/custom_artifacts.py b/src/powercontext/builtin/persistence/custom_artifacts.py index e63e0e2c73..976c8819f0 100644 --- a/src/powercontext/builtin/persistence/custom_artifacts.py +++ b/src/powercontext/builtin/persistence/custom_artifacts.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Durable declarations and JSON storage for user-defined Artifact families.""" diff --git a/src/powercontext/builtin/persistence/processing_migration.py b/src/powercontext/builtin/persistence/processing_migration.py index 15e5fad831..9c17544769 100644 --- a/src/powercontext/builtin/persistence/processing_migration.py +++ b/src/powercontext/builtin/persistence/processing_migration.py @@ -297,6 +297,9 @@ async def apply_processing_migration( # noqa: C901 - explicit durable maintenan # Names/definitions are fixed constants, never user SQL. await connection.exec_driver_sql(f"ALTER TABLE {STATES.name} ADD COLUMN {name} {definition}") return ProcessingMigrationProgress(phase="schema") + from powercontext.builtin.persistence.custom_artifacts import ensure_artifact_plugin_schema + + await ensure_artifact_plugin_schema(connection) problems = await _schema_issues(connection) if problems: raise ProcessingSchemaNotReadyError(reason="; ".join(problems)) diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index e1eb513f66..b486d0a918 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -34,7 +34,7 @@ from sqlalchemy.engine import URL, AdaptedConnection, ExceptionContext, make_url from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.exc import OperationalError -from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine from sqlalchemy.pool import ConnectionPoolEntry, StaticPool from powercontext.builtin.persistence.database import AsyncDatabase @@ -207,6 +207,7 @@ def set_pragmas(dbapi_connection: DBAPIConnection, connection_record: Connection connection_record.info["_powercontext_sqlite_stop"] = _make_sqlite_stop_idempotent(driver) if load_vector_extension: dbapi_connection.run_async(_load_sqlite_vec) + connection_record.info["powercontext_sqlite_vec_loaded"] = True cursor = dbapi_connection.cursor() try: cursor.execute(f"PRAGMA busy_timeout = {config.busy_timeout_ms}") @@ -271,6 +272,14 @@ def stop_once() -> asyncio.Future[Any] | None: return stop_once +async def ensure_sqlite_vector_extension(connection: AsyncConnection) -> None: + """Load vector functions only when this connection first serves vector search.""" + if not connection.info.get("powercontext_sqlite_vec_loaded"): + raw = await connection.get_raw_connection() + await _load_sqlite_vec(cast(Connection, raw.driver_connection)) + connection.info["powercontext_sqlite_vec_loaded"] = True + + async def _load_sqlite_vec(connection: Connection) -> None: await connection.enable_load_extension(True) try: diff --git a/src/powercontext/builtin/runtime/artifact_plugins.py b/src/powercontext/builtin/runtime/artifact_plugins.py index bfc057c23a..f63f46f267 100644 --- a/src/powercontext/builtin/runtime/artifact_plugins.py +++ b/src/powercontext/builtin/runtime/artifact_plugins.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Execute validated Artifact plugins in an owned generation Worker.""" diff --git a/src/powercontext/builtin/runtime/artifact_reads.py b/src/powercontext/builtin/runtime/artifact_reads.py index cf67b8ec3d..ff36c046c0 100644 --- a/src/powercontext/builtin/runtime/artifact_reads.py +++ b/src/powercontext/builtin/runtime/artifact_reads.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Invocation-scoped Artifact reads and shared custom-family FTS adapters.""" diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index 17a9bbba13..25ff4f7719 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -1167,7 +1167,7 @@ async def open_builtin_contexts( async with SQLiteProfile.open( database, tables=BUILTIN_TABLES + topic_index.tables + atomic_index.tables, - load_vector_extension=True, + load_vector_extension=embedding_model is not None, ) as profile: async with profile.database.transaction() as connection: # This block is SQLite-only. Reserve SQLite's single-writer slot diff --git a/src/powercontext/builtin/runtime/custom_artifact_search.py b/src/powercontext/builtin/runtime/custom_artifact_search.py index 1d840393fa..497d5f150a 100644 --- a/src/powercontext/builtin/runtime/custom_artifact_search.py +++ b/src/powercontext/builtin/runtime/custom_artifact_search.py @@ -1,13 +1,23 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Declaration-driven text/vector/hybrid search returning complete exact Artifacts.""" from __future__ import annotations import math +import sqlite3 import struct from contextlib import AsyncExitStack from dataclasses import dataclass @@ -51,6 +61,7 @@ vector_bytes, ) from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository +from powercontext.builtin.persistence.sqlite.profile import ensure_sqlite_vector_extension from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACT_PROCESSING_BINDING_STATES_TABLE from powercontext.server.authz import AccessAction, ResourceRef @@ -422,6 +433,11 @@ async def _text(self, connection, scope, build_id, index_def_id, query): return [(row.artifact_id, int(row.revision), float(row.score)) for row in result] async def _vector(self, connection, scope, definition, build_id, vector, profile): + if connection.dialect.name == "sqlite": + try: + await ensure_sqlite_vector_extension(connection) + except (AttributeError, sqlite3.Error) as error: + raise ArtifactSearchUnavailable(self.family, "SQLite vector extension is unavailable") from error table = await projection_table(connection, definition) function = "vec_distance_l2" if connection.dialect.name == "sqlite" else "l2_distance" # Table identity comes only from a validated server-owned space, never the request. diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 9191ae8cb9..f789a6b61e 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -3890,7 +3890,7 @@ class FlushProfileResponse(BaseModel): class ArtifactCollectionItem(BaseModel): scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] - family: ArtifactReadFamily + family: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[a-z][a-z0-9-]*$")] artifact_id: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[\\x21-\\x7E]+$")] schema_version: Annotated[ StrictStr | None, Field(description="Persisted schema version for a custom Artifact family.") @@ -3937,7 +3937,7 @@ class ArtifactRevisionPage(BaseModel): class ArtifactRevision(BaseModel): scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] - family: ArtifactReadFamily + family: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[a-z][a-z0-9-]*$")] artifact_id: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[\\x21-\\x7E]+$")] schema_version: Annotated[ StrictStr | None, Field(description="Persisted schema version for a custom Artifact family.") diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index ad7d6eab93..85150a7cd3 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -3110,7 +3110,7 @@ def _transport_actor(actor: ArtifactRevisionActor | None) -> TransportAccessPrin def _artifact_revision_response(value: RuntimeArtifactRecord) -> ArtifactRevision: return ArtifactRevision( scope_id=value.scope_id, - family=ArtifactReadFamily(value.family), + family=value.family, schema_version=value.schema_version, artifact_id=value.artifact_id, revision=value.revision, @@ -3139,7 +3139,7 @@ def _artifact_created_response(value: RuntimeArtifactCreated) -> ArtifactCreated def _artifact_collection_item_response(value: RuntimeArtifactCollectionItem) -> ArtifactCollectionItem: return ArtifactCollectionItem( scope_id=value.scope_id, - family=ArtifactReadFamily(value.family), + family=value.family, schema_version=value.schema_version, artifact_id=value.artifact_id, revision=value.revision, diff --git a/src/powercontext/server/artifact_plugins.py b/src/powercontext/server/artifact_plugins.py index 1f1b8513e2..700fcf650c 100644 --- a/src/powercontext/server/artifact_plugins.py +++ b/src/powercontext/server/artifact_plugins.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Server-owned hot registration and retained reads for custom Artifact plugins.""" diff --git a/tests/builtin/persistence/test_artifact_indexes.py b/tests/builtin/persistence/test_artifact_indexes.py index f4c80659ad..86ff50a771 100644 --- a/tests/builtin/persistence/test_artifact_indexes.py +++ b/tests/builtin/persistence/test_artifact_indexes.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Actual SQLite search, exact evidence and atomic projection publication.""" @@ -55,9 +64,7 @@ def declaration(*, version="1", vector_field="vector", profile=PROFILE): @asynccontextmanager async def store(config=None, model=None): - async with SQLiteProfile.open( - config or SQLiteConfig(), tables=BUILTIN_TABLES, load_vector_extension=True - ) as storage: + async with SQLiteProfile.open(config or SQLiteConfig(), tables=BUILTIN_TABLES) as storage: yield ( storage.database, ArtifactIndexBuilder(storage.database, model), @@ -105,6 +112,37 @@ def service(db, family, model=None): return result +def test_missing_vector_extension_keeps_text_search_and_content_writes_available(monkeypatch, tmp_path): + import sqlite_vec + + async def run(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'vectors.db'}") + async with store(config) as (db, builder, publisher): + await register(db, "reports", declaration()) + await write(db, publisher, "scope", "reports", "a", body("数据库", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + + monkeypatch.setattr(sqlite_vec, "loadable_path", lambda: str(tmp_path / "missing-sqlite-vec")) + async with store(config) as (db, _, publisher): + search = service(db, "reports") + with pytest.raises(ArtifactSearchUnavailable): + await search.search( + "scope", + "reports", + { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + }, + ) + result = await search.search("scope", "reports", {"mode": "text", "query": "数据库"}) + assert [artifact.artifact_id for artifact in result.artifacts] == ["a"] + revised = await write(db, publisher, "scope", "reports", "a", body("数据库 restored", [0, 1])) + assert revised.revision == 2 + + asyncio.run(run()) + + def test_exact_vectors_collapse_fragments_before_limit_and_preserve_scope_history(): async def run(): async with store() as (db, builder, publisher): diff --git a/tests/builtin/persistence/test_custom_artifact_search.py b/tests/builtin/persistence/test_custom_artifact_search.py index 1b494495e5..8294ab9709 100644 --- a/tests/builtin/persistence/test_custom_artifact_search.py +++ b/tests/builtin/persistence/test_custom_artifact_search.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Initialize missing search tables in existing Artifact storage, preserving exact historical content.""" diff --git a/tests/builtin/runtime/test_artifact_read_context.py b/tests/builtin/runtime/test_artifact_read_context.py index 5f69441cc4..c09cd8207f 100644 --- a/tests/builtin/runtime/test_artifact_read_context.py +++ b/tests/builtin/runtime/test_artifact_read_context.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """The shared read context preserves scope, family and caller permissions.""" diff --git a/tests/conftest.py b/tests/conftest.py index 91bda3b638..e3119b9416 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,19 @@ def short_tmp_path() -> Iterator[Path]: yield Path(directory) +@pytest.fixture(autouse=True) +def isolated_artifact_family_profiles(): + """Plugin registration in one test must not change another Server's readiness.""" + from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES + + original = dict(ARTIFACT_FAMILY_PROFILES) + try: + yield + finally: + ARTIFACT_FAMILY_PROFILES.clear() + ARTIFACT_FAMILY_PROFILES.update(original) + + @pytest.fixture(autouse=True) def isolated_client_connection_settings(tmp_path, monkeypatch, request): """Isolate daily runtime settings and persistent setup consent, preserving explicit real E2E runs.""" diff --git a/tests/e2e/test_artifact_index_backends.py b/tests/e2e/test_artifact_index_backends.py index 5671056c63..cf95d858d4 100644 --- a/tests/e2e/test_artifact_index_backends.py +++ b/tests/e2e/test_artifact_index_backends.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Real backend projection transactions, native initialization and exact search.""" diff --git a/tests/e2e/test_artifact_plugin_search.py b/tests/e2e/test_artifact_plugin_search.py index 9de8ae5d50..25ad575733 100644 --- a/tests/e2e/test_artifact_plugin_search.py +++ b/tests/e2e/test_artifact_plugin_search.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Custom Artifact search through generation, revisions, unload and index recovery.""" diff --git a/tests/e2e/test_artifact_plugins.py b/tests/e2e/test_artifact_plugins.py index 95ba908d24..85df6a2f28 100644 --- a/tests/e2e/test_artifact_plugins.py +++ b/tests/e2e/test_artifact_plugins.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Use a delivered plugin through the installed Server CLI and public HTTP APIs.""" diff --git a/tests/e2e/test_artifact_vector_http.py b/tests/e2e/test_artifact_vector_http.py index 285c46f829..68159b8598 100644 --- a/tests/e2e/test_artifact_vector_http.py +++ b/tests/e2e/test_artifact_vector_http.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Installed Server and spawned generation Worker share text/vector/hybrid search.""" diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py index 71b85ddee8..4e5b21549b 100644 --- a/tests/test_api_contract.py +++ b/tests/test_api_contract.py @@ -1026,11 +1026,30 @@ def test_standard_artifact_reads_share_topic_memory_family_and_display_metadata( "published_at": "2026-09-10T00:00:00Z", "source_count": 2, }) - assert item.family.root == "topic-memory" + assert item.family == "topic-memory" assert item.title == "A topic" assert item.source_count == 2 +@pytest.mark.parametrize("model", [http_models.ArtifactCollectionItem, ArtifactRevision]) +@pytest.mark.parametrize("family", ["atomic-memory", "incident-report"]) +def test_artifact_read_family_remains_a_string_with_contract_validation(model, family): + value = { + "scope_id": "scope", + "family": family, + "artifact_id": "artifact", + "revision": 1, + "content": {}, + "sources": [], + "artifacts": [], + "content_digest": f"sha256:{'0' * 64}", + } + assert model.model_validate(value).family == family + for invalid in ("", "INVALID", "x" * 129, 123): + with pytest.raises(ValidationError): + model.model_validate({**value, "family": invalid}) + + def test_base_access_uses_a_dedicated_source_type_reference() -> None: contract = yaml.safe_load(CONTRACT_PATH.read_text()) schemas = contract["components"]["schemas"] diff --git a/tests/test_artifact_index_maintenance.py b/tests/test_artifact_index_maintenance.py index 30fe78af3e..eb34552420 100644 --- a/tests/test_artifact_index_maintenance.py +++ b/tests/test_artifact_index_maintenance.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Idle index maintenance must not require SQLite's writer lock or delay startup.""" diff --git a/tests/test_artifact_plugin_leases.py b/tests/test_artifact_plugin_leases.py index 1bcdae9e1e..ebb5104132 100644 --- a/tests/test_artifact_plugin_leases.py +++ b/tests/test_artifact_plugin_leases.py @@ -1,7 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Plugin activation preserves the renewable Supervisor ownership contract.""" diff --git a/tests/test_artifact_plugin_validation.py b/tests/test_artifact_plugin_validation.py index 709b7b55f0..5ac51ef99d 100644 --- a/tests/test_artifact_plugin_validation.py +++ b/tests/test_artifact_plugin_validation.py @@ -1,9 +1,16 @@ # Copyright (c) 2026 OceanBase. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Declaration validation in a real child within its existing startup budget.""" From 2ac6c6041662664a2ca675fa7bff7d59ec460a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Tue, 27 Oct 2026 16:11:01 +0800 Subject: [PATCH 04/10] fix(artifacts): reuse scope authorization for plugin evidence --- .../builtin/runtime/artifact_plugins.py | 16 +- .../builtin/runtime/artifact_reads.py | 11 +- .../builtin/runtime/custom_artifact_search.py | 4 +- .../runtime/test_artifact_read_context.py | 210 +++++++++++++++++- 4 files changed, 219 insertions(+), 22 deletions(-) diff --git a/src/powercontext/builtin/runtime/artifact_plugins.py b/src/powercontext/builtin/runtime/artifact_plugins.py index f63f46f267..1eb3123331 100644 --- a/src/powercontext/builtin/runtime/artifact_plugins.py +++ b/src/powercontext/builtin/runtime/artifact_plugins.py @@ -113,7 +113,6 @@ async def _restore_access_families(database) -> None: async def _process_window(spec: ArtifactPluginWorkerSpec, assignment: ArtifactProcessingWorkAssignment): # noqa: C901 - One authorized window with preparation outside its commit. # Server authorization is reconstructed in this child using deployment identity. - from powercontext.server.authz import AccessAction, ResourceRef from powercontext.server.processing_security import open_worker_security definition = spec.definition @@ -146,12 +145,6 @@ async def authorize(connection): await require_current_binding(connection, spec, assignment.scope_id) if security is not None: await security.authorize_transaction(connection, assignment.scope_id) - await security.access.with_connection(connection).require( - security.principal, - AccessAction.SCOPE_READ, - ResourceRef.scope(assignment.scope_id), - context=security.context, - ) invocation = ScopeInvocation(assignment, authorize_transaction=authorize) try: @@ -304,18 +297,11 @@ async def _commit_window( validated = _validate_drafts(declaration, source_evidence, artifact_evidence, drafts) repository = custom_artifact_repository(declaration["family"], sources, prepared=prepared_indexes) async with database.transaction() as connection: + # The Worker guard rechecks Scope contribution, which includes evidence reads. await invocation.guard(connection) for draft in validated: resource = ResourceRef.artifact(assignment.scope_id, family=draft.family, artifact_id=draft.artifact_id) bound = None if security is None else security.access.with_connection(connection) - if bound is not None: - for ref in draft.artifact_refs: - await bound.require( - security.principal, - AccessAction.ARTIFACT_READ, - ResourceRef.artifact(assignment.scope_id, family=ref.family, artifact_id=ref.artifact_id), - context=security.context, - ) prepared = RepositoryArtifactDraft( family=draft.family, content=StoredArtifactContent(schema_version=draft.schema_version, content=draft.content), diff --git a/src/powercontext/builtin/runtime/artifact_reads.py b/src/powercontext/builtin/runtime/artifact_reads.py index ff36c046c0..c5d21caee7 100644 --- a/src/powercontext/builtin/runtime/artifact_reads.py +++ b/src/powercontext/builtin/runtime/artifact_reads.py @@ -33,7 +33,7 @@ from powercontext.builtin.records import BaseArtifactFamily, BaseValueNotFoundError, InvalidBaseAccessRequestError from powercontext.builtin.runtime.artifact_search import ArtifactSearchService from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher -from powercontext.server.authz import AccessAction, ResourceRef +from powercontext.server.authz import AccessAction, BuiltinAuthorizationProvider, ResourceRef class _StoredJsonContent(RootModel[dict[str, JsonValue]]): @@ -59,6 +59,8 @@ def __init__( self.database, self.scope_id = database, scope_id self.families = frozenset(families) self.access, self.principal, self.audit, self.guard = access, principal, audit, guard + # Only the exact built-in Provider guarantees Scope-to-Artifact read inheritance. + self._scope_covers_artifacts = access is None or type(access.provider) is BuiltinAuthorizationProvider self.refs: set[tuple[str, str, int]] = set() self.cursor = SignedCursorCodec() self.search_service = search_service or ArtifactSearchService( @@ -80,7 +82,8 @@ async def _connection(self, family): await connection.exec_driver_sql("SAVEPOINT pc_artifact_read") if self.guard is not None: await self.guard(connection) - if self.access is not None: + # The Worker guard already checks current Scope contribution in this transaction. + if self.access is not None and (self.guard is None or not self._scope_covers_artifacts): await self.access.for_read(connection).require( self.principal, AccessAction.SCOPE_READ, @@ -121,7 +124,7 @@ async def read(self, family, artifact_id, revision): if revision is None: return None raise BaseValueNotFoundError("artifact_revision", (family, artifact_id, revision)) from None - if self.access is not None: + if self.access is not None and not self._scope_covers_artifacts and family != "topic-memory": await self.access.for_read(connection).require( self.principal, AccessAction.ARTIFACT_READ, @@ -167,7 +170,7 @@ async def list(self, family, *, limit=50, cursor=None): break for artifact_id, revision in rows: after = artifact_id - if bound is not None: + if bound is not None and not self._scope_covers_artifacts and family != "topic-memory": decision = await bound.check( self.principal, AccessAction.ARTIFACT_READ, diff --git a/src/powercontext/builtin/runtime/custom_artifact_search.py b/src/powercontext/builtin/runtime/custom_artifact_search.py index 497d5f150a..7a076c22fa 100644 --- a/src/powercontext/builtin/runtime/custom_artifact_search.py +++ b/src/powercontext/builtin/runtime/custom_artifact_search.py @@ -63,7 +63,7 @@ from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository from powercontext.builtin.persistence.sqlite.profile import ensure_sqlite_vector_extension from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACT_PROCESSING_BINDING_STATES_TABLE -from powercontext.server.authz import AccessAction, ResourceRef +from powercontext.server.authz import AccessAction, BuiltinAuthorizationProvider, ResourceRef class ArtifactQueryVector(BaseModel): @@ -292,6 +292,8 @@ async def _candidates(self, connection, scope_id, request, mode, selected, query if identity not in visible: visible[identity] = ( bound is None + # The same transaction already authorized this Scope's reads. + or type(bound.provider) is BuiltinAuthorizationProvider or ( await bound.check( execution.principal, diff --git a/tests/builtin/runtime/test_artifact_read_context.py b/tests/builtin/runtime/test_artifact_read_context.py index c09cd8207f..15cc39ac4f 100644 --- a/tests/builtin/runtime/test_artifact_read_context.py +++ b/tests/builtin/runtime/test_artifact_read_context.py @@ -17,26 +17,36 @@ import asyncio from contextlib import AsyncExitStack from datetime import UTC, datetime +from functools import partial from typing import cast import pytest from sqlalchemy import insert, select -from powercontext.artifact_plugins import GenerationContext +from powercontext.artifact_plugins import ArtifactDraft, GenerationContext from powercontext.artifacts import ArtifactRef from powercontext.artifacts.search import ArtifactSearchUnsupported from powercontext.builtin.artifacts.profile.models import Profile, ProfileContent, ProfileGeneration +from powercontext.builtin.artifacts.topic_memory.models import TopicMemory, TopicMemoryContent from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder from powercontext.builtin.persistence.artifacts import ArtifactRepository, RepositoryArtifactDraft from powercontext.builtin.persistence.custom_artifacts import ( StoredArtifactContent, custom_artifact_repository, ) +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.sqlite.experience_index import SQLiteExperienceFTSIndex +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE, BUILTIN_TABLES, SOURCE_CURSORS_TABLE from powercontext.builtin.records import BaseValueNotFoundError, InvalidBaseAccessRequestError +from powercontext.builtin.runtime.artifact_plugins import ArtifactPluginWorkerSpec, _commit_window from powercontext.builtin.runtime.artifact_reads import ArtifactReadSession +from powercontext.builtin.runtime.processing_contracts import ArtifactProcessingWorkAssignment +from powercontext.builtin.runtime.processing_execution import ScopeInvocation +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY from powercontext.server.authz import ( AccessAuditContext, AccessControlService, @@ -51,6 +61,7 @@ from powercontext.server.authz.composition import open_builtin_access_control from powercontext.server.authz.profiles import register_custom_artifact_access_family from powercontext.server.authz.repository import RelationalAccessRepository +from powercontext.server.processing_security import WorkerSecurity class FilteredProvider(BuiltinAuthorizationProvider): @@ -90,6 +101,15 @@ async def scenario(): repository = custom_artifact_repository(family) async with profile.database.transaction() as connection: await SQLiteExperienceFTSIndex().initialize(connection) + topic = await ArtifactRepository((TopicMemory,)).create( + connection, + "scope", + "topic", + RepositoryArtifactDraft( + family="topic-memory", + content=TopicMemoryContent(title="Topic", summary="Summary", detail="Evidence"), + ), + ) await connection.execute( insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( family=family, @@ -117,13 +137,22 @@ async def scenario(): idempotency_key=identity, context=audit, ) + builtin_reads = ArtifactReadSession( + profile.database, "scope", (family,), access=access, principal=alice, audit=audit + ) + builtin_page = await builtin_reads.list(family) + assert {ref.artifact_id for ref in builtin_page.items} == {"a-hidden", "b-visible"} + assert (await builtin_reads.read(family, "a-hidden", 1)).content["text"] == "数据库 TIMEOUT" + builtin_matches = await builtin_reads.search(family, "数据库", limit=2) + assert {hit.artifact_ref.artifact_id for hit in builtin_matches.matches} == {"a-hidden", "b-visible"} + # An injected policy may narrow object visibility; it is not the Worker provider. access = AccessControlService( FilteredProvider(cast(RelationalAccessRepository, access.relationships)), relationships=access.relationships, audit=access.audit, ) reads = ArtifactReadSession( - profile.database, "scope", (family,), access=access, principal=alice, audit=audit + profile.database, "scope", (family, "topic-memory"), access=access, principal=alice, audit=audit ) context = GenerationContext( "scope", resources, reads.read, list_artifacts=reads.list, search_artifacts=reads.search @@ -160,6 +189,17 @@ async def scenario(): await foreign_context.list_artifacts(family) with pytest.raises(AccessDeniedError): await foreign_context.search_artifacts(family, "数据库") + topic_page = await context.list_artifacts("topic-memory") + assert topic_page.items == (topic.as_ref(),) + assert (await context.read_revision(topic.as_ref())).content["detail"] == "Evidence" + assert ("topic-memory", "topic", 1) in reads.refs + unauthorized = ArtifactReadSession( + profile.database, "scope", ("topic-memory",), access=access, principal=bob, audit=audit + ) + with pytest.raises(AccessDeniedError): + await unauthorized.list("topic-memory") + with pytest.raises(AccessDeniedError): + await unauthorized.read("topic-memory", "topic", 1) asyncio.run(scenario()) @@ -274,3 +314,169 @@ async def scenario(): assert cursor["cursor"] == b'{"through":7}' and cursor["generation"] == 3 asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "evidence_family,revoked,update_foreign", + [ + ("topic-memory", False, False), + ("topic-memory", True, False), + ("profile", False, False), + ("profile", True, False), + ("topic-memory", False, True), + ], +) +def test_plugin_commit_rechecks_evidence_permissions(tmp_path, evidence_family, revoked, update_foreign): + async def scenario(): + admin = PrincipalRef(type="service", id="admin") + alice = PrincipalRef(type="user", id="alice") + audit = AccessAuditContext(transport="test", operation="artifact-reads") + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'commit.db'}") + output_family = "context-reports" + register_custom_artifact_access_family(output_family) + async with AsyncExitStack() as resources: + profile = await resources.enter_async_context(SQLiteProfile.open(config, tables=BUILTIN_TABLES)) + access = await resources.enter_async_context( + open_builtin_access_control(config, bootstrap_administrators=(admin,)) + ) + binding = await access.create_binding( + admin, + CreateBinding( + subject=alice, + resource=ResourceRef.scope("scope"), + role=AccessRole.SCOPE_CONTRIBUTOR, + idempotency_key="scope-viewer", + ), + context=audit, + ) + async with profile.database.transaction() as connection: + if evidence_family == "topic-memory": + artifact_type = TopicMemory + content = TopicMemoryContent(title="Topic", summary="Summary", detail="Evidence") + else: + artifact_type = Profile + content = ProfileContent( + content="Evidence", + generation=ProfileGeneration(mode="manual_create", created_at=datetime.now(UTC)), + ) + evidence = await ArtifactRepository((artifact_type,)).create( + connection, + "scope", + "a-hidden", + RepositoryArtifactDraft(family=evidence_family, content=content), + ) + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=output_family, + schema_version="1", + declaration={"schema_version": "1"}, + ) + ) + lease = await ArtifactProcessingLeaseRepository().start_single_process_term(connection, "test-owner") + intent = await ArtifactProcessingIntentRepository().request(connection, "scope", "reports") + assignment = ArtifactProcessingWorkAssignment( + binding_name="reports", + scope_id="scope", + artifact_family=output_family, + claimed_request_generation=intent.requested_generation, + fence=lease.fence("single-process"), + worker_id="test-worker", + ) + security = WorkerSecurity(access, alice) + invocation = ScopeInvocation( + assignment, authorize_transaction=partial(security.authorize_transaction, scope_id="scope") + ) + await invocation.start(connection) + if evidence_family == "profile": + await access.establish_artifact_owner( + ResourceRef.artifact("scope", family=evidence_family, artifact_id="a-hidden"), + alice, + idempotency_key="evidence-owner", + context=audit, + ) + reads = ArtifactReadSession( + profile.database, + "scope", + (evidence_family, output_family), + access=access, + principal=alice, + audit=audit, + guard=invocation.guard, + ) + snapshot = await reads.read(evidence_family, "a-hidden", 1) + assert snapshot.ref == evidence.as_ref() + if revoked: + await access.revoke_binding( + admin, binding.binding_id, expected_version=1, idempotency_key="revoke-scope", context=audit + ) + with pytest.raises(AccessDeniedError): + await reads.read(evidence_family, "a-hidden", 1) + with pytest.raises(AccessDeniedError): + await reads.list(evidence_family) + if update_foreign: + async with profile.database.transaction() as connection: + await custom_artifact_repository(output_family).create( + connection, + "scope", + "report", + RepositoryArtifactDraft( + family=output_family, + content=StoredArtifactContent(schema_version="1", content={"text": "Original"}), + ), + ) + await access.establish_artifact_owner( + ResourceRef.artifact("scope", family=output_family, artifact_id="report"), + PrincipalRef(type="user", id="bob"), + idempotency_key="output-owner", + context=audit, + ) + previous = await reads.read(output_family, "report", 1) + assert previous.content == {"text": "Original"} + draft = ArtifactDraft( + family=output_family, + schema_version="1", + artifact_id="report", + expected_revision=1 if update_foreign else 0, + content={"text": "Result"}, + source_refs=(), + artifact_refs=(snapshot.ref,), + ) + + async def commit(): + await _commit_window( + profile.database, + ArtifactPluginWorkerSpec(config, "reports", 1, {}), + assignment, + {"family": output_family, "schema_version": "1", "content_schema": {"type": "object"}}, + invocation, + SourceRepository(BUILTIN_SOURCE_REGISTRY), + None, + 0, + frozenset(), + reads.refs, + [draft], + security, + ) + + if revoked or update_foreign: + with pytest.raises(AccessDeniedError): + await commit() + else: + await commit() + async with profile.database.transaction() as connection: + repository = custom_artifact_repository(output_family) + if revoked: + with pytest.raises(RepositoryNotFoundError): + await repository.latest(connection, "scope", output_family, "report") + else: + result = await repository.latest(connection, "scope", output_family, "report") + if update_foreign: + assert result.revision == 1 + assert result.content.content == {"text": "Original"} + else: + assert result.lineage.artifacts == (snapshot.ref,) + current = await ArtifactProcessingIntentRepository().load(connection, "scope", "reports") + assert current is not None + assert current.handled_generation == (0 if revoked or update_foreign else intent.requested_generation) + + asyncio.run(scenario()) From 9fe2e68220a6e0aa8ebe4b78900cd12421db5cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Tue, 27 Oct 2026 17:24:45 +0800 Subject: [PATCH 05/10] fix(migrations): batch artifact registration columns and update review --- release/upgrade-support.json | 4 +- .../schemas/mysql-v1_5_0_001.json | 924 +----------------- .../test_artifact_storage_lifecycle.py | 25 + 3 files changed, 28 insertions(+), 925 deletions(-) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index c0afba9bf2..89bebd30a9 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.1", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "9b67d639cf57b3ae7cf37dcd1845f50561af36bb19edd9a7b979408f31e9a6de", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB full-server-code and full-server-vector-code are temporarily manual-only due to native DDL waits with the available engine. Main and publication require the other ten; no manual result is needed for publication. The two deferred configurations remain unverified and are excluded from generated supported upgrade declarations. Reports retain their identities and explicitly record incomplete full-matrix acceptance. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged." + "persistence_sha256": "53c53c51314d5aca02ea4d5e25b7c54299a433eb68932f0c26b7be92fe215498", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB full-server-code and full-server-vector-code are temporarily manual-only due to native DDL waits with the available engine. Main and publication require the other ten; no manual result is needed for publication. The two deferred configurations remain unverified and are excluded from generated supported upgrade declarations. Reports retain their identities and explicitly record incomplete full-matrix acceptance. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Artifact plugin storage extends the shared Server chain from s1402 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_001.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_001.json index 997b60f1e4..122c3b914d 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_001.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_001.json @@ -3,13 +3,7 @@ "revisions": { "v1_5_0_001": { "statements": [ - "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_id VARCHAR(128)", - "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_family VARCHAR(128)", - "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_spec JSON", - "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN binding_generation BIGINT NOT NULL DEFAULT '0'", - "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN desired_state VARCHAR(16)", - "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN phase VARCHAR(16)", - "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN last_error VARCHAR(1024)", + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_id VARCHAR(128), ADD COLUMN plugin_family VARCHAR(128), ADD COLUMN plugin_spec JSON, ADD COLUMN binding_generation BIGINT NOT NULL DEFAULT '0', ADD COLUMN desired_state VARCHAR(16), ADD COLUMN phase VARCHAR(16), ADD COLUMN last_error VARCHAR(1024)", "CREATE TABLE pc_artifact_family_schemas (\n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tschema_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdeclaration JSON NOT NULL, \n\tPRIMARY KEY (family, schema_version)\n)", "CREATE TABLE pc_artifact_vector_spaces (\n\tspace_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprofile_fingerprint VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprofile_json JSON NOT NULL, \n\tdimension INTEGER NOT NULL, \n\tdistance VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tnormalization VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tstorage_format_version VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tbackend VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tphysical_table VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprovision_state VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (space_id), \n\tCONSTRAINT uq_pc_artifact_vector_space UNIQUE (backend, profile_fingerprint, storage_format_version), \n\tCONSTRAINT ck_pc_artifact_vector_dimension CHECK (dimension > 0)\n)", "CREATE TABLE pc_artifact_vector_indexes (\n\tindex_def_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tindex_name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tschema_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprojection_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tspace_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tsource_kind VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprojection_spec JSON NOT NULL, \n\tdefinition_fingerprint VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (index_def_id), \n\tCONSTRAINT uq_pc_artifact_index_definition UNIQUE (family, index_name, schema_version, projection_version, space_id, definition_fingerprint), \n\tFOREIGN KEY(space_id) REFERENCES pc_artifact_vector_spaces (space_id), \n\tFOREIGN KEY(family, schema_version) REFERENCES pc_artifact_family_schemas (family, schema_version)\n)", @@ -21,922 +15,6 @@ "CREATE FULLTEXT INDEX ix_pc_artifact_search_fts ON pc_artifact_search_text (searchable_text) WITH PARSER SPACE" ], "changes": [ - { - "pc_artifact_processing_binding_states": { - "columns": { - "binding_name": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "last_auto_wave_completed_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "last_schedule_checkpoint_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "scan_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_in_progress": { - "type": [ - "tinyint", - [ - "1" - ] - ], - "nullable": false, - "default": [ - "literal", - "0" - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_upper_pending_sequence": { - "type": [ - "bigint", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_id": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "binding_name", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": { - "ck_pc_processing_scan_generation": { - "expression": [ - "binary", - ">=", - [ - "column", - "scan_generation" - ], - [ - "number", - 0 - ] - ], - "enforced": true - } - }, - "indexes": {} - } - }, - { - "pc_artifact_processing_binding_states": { - "columns": { - "binding_name": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "last_auto_wave_completed_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "last_schedule_checkpoint_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "scan_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_in_progress": { - "type": [ - "tinyint", - [ - "1" - ] - ], - "nullable": false, - "default": [ - "literal", - "0" - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_upper_pending_sequence": { - "type": [ - "bigint", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_id": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_family": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "binding_name", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": { - "ck_pc_processing_scan_generation": { - "expression": [ - "binary", - ">=", - [ - "column", - "scan_generation" - ], - [ - "number", - 0 - ] - ], - "enforced": true - } - }, - "indexes": {} - } - }, - { - "pc_artifact_processing_binding_states": { - "columns": { - "binding_name": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "last_auto_wave_completed_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "last_schedule_checkpoint_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "scan_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_in_progress": { - "type": [ - "tinyint", - [ - "1" - ] - ], - "nullable": false, - "default": [ - "literal", - "0" - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_upper_pending_sequence": { - "type": [ - "bigint", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_id": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_family": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_spec": { - "type": [ - "json", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "binding_name", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": { - "ck_pc_processing_scan_generation": { - "expression": [ - "binary", - ">=", - [ - "column", - "scan_generation" - ], - [ - "number", - 0 - ] - ], - "enforced": true - } - }, - "indexes": {} - } - }, - { - "pc_artifact_processing_binding_states": { - "columns": { - "binding_name": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "last_auto_wave_completed_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "last_schedule_checkpoint_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "scan_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_in_progress": { - "type": [ - "tinyint", - [ - "1" - ] - ], - "nullable": false, - "default": [ - "literal", - "0" - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_upper_pending_sequence": { - "type": [ - "bigint", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_id": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_family": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_spec": { - "type": [ - "json", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "binding_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "binding_name", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": { - "ck_pc_processing_scan_generation": { - "expression": [ - "binary", - ">=", - [ - "column", - "scan_generation" - ], - [ - "number", - 0 - ] - ], - "enforced": true - } - }, - "indexes": {} - } - }, - { - "pc_artifact_processing_binding_states": { - "columns": { - "binding_name": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "last_auto_wave_completed_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "last_schedule_checkpoint_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "scan_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_in_progress": { - "type": [ - "tinyint", - [ - "1" - ] - ], - "nullable": false, - "default": [ - "literal", - "0" - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_upper_pending_sequence": { - "type": [ - "bigint", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_id": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_family": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_spec": { - "type": [ - "json", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "binding_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "desired_state": { - "type": [ - "varchar", - [ - "16" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "binding_name", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": { - "ck_pc_processing_scan_generation": { - "expression": [ - "binary", - ">=", - [ - "column", - "scan_generation" - ], - [ - "number", - 0 - ] - ], - "enforced": true - } - }, - "indexes": {} - } - }, - { - "pc_artifact_processing_binding_states": { - "columns": { - "binding_name": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "last_auto_wave_completed_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "last_schedule_checkpoint_at": { - "type": [ - "datetime", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "scan_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_in_progress": { - "type": [ - "tinyint", - [ - "1" - ] - ], - "nullable": false, - "default": [ - "literal", - "0" - ], - "charset": null, - "collation": null, - "extra": [] - }, - "scan_upper_pending_sequence": { - "type": [ - "bigint", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_id": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_family": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "plugin_spec": { - "type": [ - "json", - [] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "binding_generation": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": [ - "number", - 0 - ], - "charset": null, - "collation": null, - "extra": [] - }, - "desired_state": { - "type": [ - "varchar", - [ - "16" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "phase": { - "type": [ - "varchar", - [ - "16" - ] - ], - "nullable": true, - "default": null, - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "binding_name", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": { - "ck_pc_processing_scan_generation": { - "expression": [ - "binary", - ">=", - [ - "column", - "scan_generation" - ], - [ - "number", - 0 - ] - ], - "enforced": true - } - }, - "indexes": {} - } - }, { "pc_artifact_processing_binding_states": { "columns": { diff --git a/tests/builtin/persistence/test_artifact_storage_lifecycle.py b/tests/builtin/persistence/test_artifact_storage_lifecycle.py index 2d9cf7dc42..a109789dfe 100644 --- a/tests/builtin/persistence/test_artifact_storage_lifecycle.py +++ b/tests/builtin/persistence/test_artifact_storage_lifecycle.py @@ -45,11 +45,36 @@ from tests.builtin.persistence.test_mysql_migration_runner import ( _apply, _full_server_runner, + _interrupt_after_ddl, backend, # noqa: F401 - Real disposable backend fixture. ) from tests.e2e.atomic_memory_migration_backend import migration_profile +def test_fixed_artifact_storage_resumes_after_registration_columns(backend): # noqa: F811 + runner = _full_server_runner(backend) + _interrupt_after_ddl( + runner, + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_id", + policy="auto", + ) + (journal,) = backend.evidence_directory.rglob("maintenance.json") + interrupted = json.loads(journal.read_text()) + assert interrupted["last_revision"] == "s1402" + assert interrupted["pending_revision"] == "v1_5_0_001" + recovery = _full_server_runner(backend).plan() + assert recovery.state == "recovery_required" and recovery.source_revision == "s1402" + # The DDL returned and its owned connection closed before the injected failure. + resumed = _full_server_runner(backend, acknowledge_previous_run=(recovery.coordination or {}).get("pending_run_id")) + result = _apply(resumed) + assert result.state == "ready" and result.revision == resumed.bundle.head + assert resumed.verify().state == "ready" + completed = json.loads(journal.read_text()) + assert completed["state"] == "complete" + assert completed["maintenance_window_id"] == interrupted["maintenance_window_id"] + assert completed["source_fingerprint"] == interrupted["source_fingerprint"] + + def _next_fixed_revision(bundle, directory): """A subsequent fixed-table addition leaves the space format untouched.""" shutil.copytree(bundle.directory, directory) From b2af2aa5ac1609004e913e10c134e71916795986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Wed, 28 Oct 2026 12:19:33 +0800 Subject: [PATCH 06/10] fix(server): restore artifact authorization on startup --- src/powercontext/server/artifact_plugins.py | 2 - src/powercontext/server/factory.py | 18 ++- .../test_artifact_authorization_startup.py | 137 ++++++++++++++++++ 3 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/test_artifact_authorization_startup.py diff --git a/src/powercontext/server/artifact_plugins.py b/src/powercontext/server/artifact_plugins.py index 6a0635afec..04d2d81341 100644 --- a/src/powercontext/server/artifact_plugins.py +++ b/src/powercontext/server/artifact_plugins.py @@ -119,8 +119,6 @@ def __init__( async def __aenter__(self): async with self.database.transaction() as connection: families = (await connection.execute(select(_SCHEMAS.c.family).distinct())).scalars().all() - for family in families: - register_custom_artifact_access_family(family) rows = (await connection.execute(select(_STATES).where(_STATES.c.plugin_id.is_not(None)))).mappings().all() restored = [] for persisted in rows: diff --git a/src/powercontext/server/factory.py b/src/powercontext/server/factory.py index ba0313fb16..bb62ba177a 100644 --- a/src/powercontext/server/factory.py +++ b/src/powercontext/server/factory.py @@ -24,6 +24,7 @@ from fastapi import FastAPI, Request, Response from fastapi.routing import APIRoute +from sqlalchemy import select from starlette.middleware import Middleware from powercontext._logging import log_safely @@ -38,6 +39,7 @@ from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.records import RelationalRecordService from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE from powercontext.builtin.records import BaseOperationNotSupportedError from powercontext.builtin.runtime import ( BuiltinRuntime, @@ -77,6 +79,7 @@ access_control_for_mode, ) from powercontext.server.authz.composition import open_builtin_access_control +from powercontext.server.authz.profiles import register_custom_artifact_access_family from powercontext.server.authz.repository import RelationalAccessRepository from powercontext.server.context import current_principal, current_request_id from powercontext.server.cursor_secret import resolve_cursor_secret @@ -274,14 +277,15 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: identity_repository = ServerIdentityRepository(runtime.primary_database) await identity_repository.initialize() server_id = await identity_repository.load_or_create() + records = runtime._records() + if not isinstance(records, RelationalRecordService): + raise TypeError("Artifact plugins require relational record storage") # noqa: TRY003 + await _restore_artifact_access_profiles(records) _bind_evidence_access(runtime, active_access_control, resolved.access.mode) if ( config.runtime.artifact_processing_supervisor_mode == "global" and config.runtime.artifact_processing_role != "api" ): - records = runtime._records() - if not isinstance(records, RelationalRecordService): - raise TypeError("Artifact plugins require relational record storage") # noqa: TRY003 def bind_plugin_supervisor(supervisors) -> None: runtime.artifact_processing_supervisor = supervisors @@ -431,6 +435,14 @@ def current_capabilities() -> Capabilities: return app +async def _restore_artifact_access_profiles(records: RelationalRecordService) -> None: + """Restore retained family access even when this process runs no plugin manager.""" + async with records.database.transaction() as connection: + families = (await connection.execute(select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family).distinct())).scalars().all() + for family in families: + register_custom_artifact_access_family(family) + + def _state_access_checks(app: FastAPI) -> Callable[[], Awaitable[Mapping[str, str]]]: async def checks() -> dict[str, str]: return await access_readiness_checks(app.state) diff --git a/tests/e2e/test_artifact_authorization_startup.py b/tests/e2e/test_artifact_authorization_startup.py new file mode 100644 index 0000000000..acc0a9da56 --- /dev/null +++ b/tests/e2e/test_artifact_authorization_startup.py @@ -0,0 +1,137 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Persisted custom artifacts remain authorized without a plugin manager.""" + +import asyncio + +import httpx +import pytest +from sqlalchemy import insert + +from powercontext.builtin.persistence.artifacts import RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE +from powercontext.builtin.runtime.config import RuntimeConfig +from powercontext.server.authentication import StaticBearerAuthenticationProvider +from powercontext.server.authz import AccessAuditContext, AccessRole, CreateBinding, PrincipalRef, ResourceRef +from powercontext.server.authz.composition import open_builtin_access_control +from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES +from powercontext.server.factory import create_server_app +from powercontext.server.settings import AccessControlConfig, DashboardConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database +from tests.e2e.test_atomic_memory_search_authorization_snapshot import _isolated_database + + +@pytest.mark.parametrize( + "backend,role,mode", + [("sqlite", "all", "global"), ("sqlite", "all", "dedicated"), ("oceanbase", "api", "global")], +) +def test_custom_artifact_authorization_survives_server_restart(tmp_path, monkeypatch, backend, role, mode): + async def scenario(): + family = "startup-report" + admin = PrincipalRef(type="service", id="admin") + reader = PrincipalRef(type="user", id="reader") + async with _isolated_database(backend, tmp_path) as database: + settings = ServerSettings( + database=database, + workspace=tmp_path, + access=AccessControlConfig(mode="enforced"), + runtime=RuntimeConfig( + artifact_processing_families=(), + artifact_processing_role=role, + artifact_processing_supervisor_mode=mode, + ), + dashboard=DashboardConfig(enabled=False), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings, evidence_dir=tmp_path / "evidence") + async with open_builtin_access_control(database, bootstrap_administrators=(admin,)) as access: + + def application(principal): + return create_server_app( + settings=settings, + scheduler_path=tmp_path / "scheduler.db", + access_control=access, + authentication_provider=StaticBearerAuthenticationProvider("test-token", principal), + ) + + app = application(admin) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + scopes = [] + for name in ("visible", "forbidden"): + response = await client.post( + "/v1/scopes", json={"title": name, "summary": name, "idempotency_key": name} + ) + assert response.status_code == 201, response.text + scopes.append(response.json()["scope_id"]) + async with app.state.application._records().database.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={"family": family, "schema_version": "1"}, + ) + ) + for scope in scopes: + await custom_artifact_repository(family).create( + connection, + scope, + "report", + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent(schema_version="1", content={"text": "retained"}), + ), + ) + await access.create_binding( + admin, + CreateBinding( + subject=reader, + resource=ResourceRef.scope(scopes[0]), + role=AccessRole.SCOPE_VIEWER, + idempotency_key="reader", + ), + context=AccessAuditContext(transport="test", operation="artifact-startup"), + ) + # Model an empty process-local registry on restart, preserving built-in profiles. + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + app = application(reader) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + path = f"/v1/scopes/{scopes[0]}/artifacts/{family}/report" + for suffix in ("", "/revisions/1"): + response = await client.get(path + suffix) + assert response.status_code == 200, response.text + assert response.json()["content"] == {"text": "retained"} + assert response.json()["revision"] == 1 + denied = await client.get(f"/v1/scopes/{scopes[1]}/artifacts/{family}/report") + assert denied.status_code == 403, denied.text + unknown = await client.get(f"/v1/scopes/{scopes[0]}/artifacts/unregistered-family/report") + assert unknown.status_code in {404, 422}, unknown.text + + asyncio.run(scenario()) From 4ddfaf22893fb7b8b5d7f4686c9293d8f00b4b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Thu, 29 Oct 2026 14:22:48 +0800 Subject: [PATCH 07/10] fix(artifact): discover and authorize retained families independently --- src/powercontext/server/app.py | 45 ++- src/powercontext/server/artifact_plugins.py | 34 +- src/powercontext/server/authz/service.py | 6 + src/powercontext/server/factory.py | 11 +- .../test_artifact_authorization_startup.py | 366 +++++++++++++++++- 5 files changed, 443 insertions(+), 19 deletions(-) diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index 28ff69f0be..96e845db7d 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -862,7 +862,11 @@ owner_recovery_session, owner_write_access, ) -from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES, artifact_family_profile +from powercontext.server.authz.profiles import ( + ARTIFACT_FAMILY_PROFILES, + artifact_family_profile, + register_custom_artifact_access_family, +) from powercontext.server.authz.service import _require_allowed from powercontext.server.context import ( bind_request_id, @@ -1381,6 +1385,8 @@ def create_app( ) app.openapi_version = OPENAPI_VERSION app.state.artifact_plugin_manager = None + app.state.artifact_declaration_provider = None + app.state.artifact_access_refresher = None app.state.application = application app.state.capability_provider = capability_provider app.state.readiness_probe = readiness_probe @@ -1690,6 +1696,7 @@ async def access_readiness_checks(state: Any) -> dict[str, str]: authentication_status = "ready" if (await authentication.readiness()).ready else "not_ready" except Exception: authentication_status = "not_ready" + # This process learns families at startup, on access, or through capabilities discovery. family_capabilities = ",".join( f"{profile.family}:{'enabled' if profile.enabled else 'disabled'}" for profile in sorted(ARTIFACT_FAMILY_PROFILES.values(), key=lambda item: item.family) @@ -1778,10 +1785,14 @@ async def unload_artifact_plugin( async def get_capabilities(request: Request) -> Capabilities: capability_provider: CapabilityProvider | None = request.app.state.capability_provider value = capability_provider() if capability_provider is not None else request.app.state.capabilities - manager = request.app.state.artifact_plugin_manager - if manager is None: + declaration_provider: Callable[[], Awaitable[list[dict[str, Any]]]] | None = ( + request.app.state.artifact_declaration_provider + ) + if declaration_provider is None: return value - declarations = await manager.declarations() + declarations = await declaration_provider() + for declaration in declarations: + register_custom_artifact_access_family(declaration["family"]) return Capabilities.model_validate({ **value.model_dump(mode="json"), "artifact_families": value.artifact_families @@ -1834,6 +1845,7 @@ async def check_access(payload: AccessCheckRequest, request: Request) -> AccessC ) for requirement in payload.requirements ]) + await _refresh_artifact_access(request.app.state, {resource.family for _, resource in requirements}) decisions = await access.check_batch( _require_principal(), requirements, @@ -1853,6 +1865,8 @@ async def check_access(payload: AccessCheckRequest, request: Request) -> AccessC async def list_access_resources(payload: ListAccessResourcesRequest, request: Request) -> AccessResourcePage: access = _require_access_control(request) resource_type = AccessResourceType(payload.resource_type.value) + if resource_type is AccessResourceType.ARTIFACT: + await _refresh_artifact_access(request.app.state, {payload.family}) page = await access.list_resources( _require_principal(), action=AccessAction(payload.action.value), @@ -1971,6 +1985,7 @@ async def list_access_roles(payload: ListAccessRolesRequest, request: Request) - if payload.family is not None: if resource_type not in {None, AccessResourceType.ARTIFACT}: raise AccessInvalidRequestError("action-resource") + await _refresh_artifact_access(request.app.state, {payload.family}) selected_profile = ARTIFACT_FAMILY_PROFILES.get(payload.family) if selected_profile is None: raise AccessInvalidRequestError("artifact-family") @@ -2024,10 +2039,12 @@ async def list_access_roles(payload: ListAccessRolesRequest, request: Request) - async def list_access_bindings(payload: ListAccessBindingsRequest, request: Request) -> AccessBindingPage: access = _require_access_control(request) + resource = _supported_access_resource(_access_resource(payload.management_resource)) + await _refresh_artifact_access(request.app.state, {resource.family}) page = await access.list_bindings( _require_principal(), BindingSearchRequest( - management_resource=_supported_access_resource(_access_resource(payload.management_resource)), + management_resource=resource, subject=None if payload.subject is None else _access_subject(payload.subject), role=None if payload.role is None else AccessRole(payload.role.value), state=None if payload.state is None else AccessBindingState(payload.state.value), @@ -2044,11 +2061,13 @@ async def list_access_bindings(payload: ListAccessBindingsRequest, request: Requ async def create_access_binding(payload: CreateAccessBindingRequest, request: Request) -> TransportAccessBinding: access = _require_access_control(request) + resource = _supported_access_resource(_access_resource(payload.resource)) + await _refresh_artifact_access(request.app.state, {resource.family}) binding = await access.create_binding( _require_principal(), CreateBinding( subject=_access_subject(payload.subject), - resource=_supported_access_resource(_access_resource(payload.resource)), + resource=resource, role=AccessRole(payload.role.value), idempotency_key=payload.idempotency_key, reason=payload.reason, @@ -2068,6 +2087,7 @@ async def revoke_access_binding(payload: RevokeAccessBindingRequest, request: Re expected_version=payload.expected_version, idempotency_key=payload.idempotency_key, context=_access_audit_context(REVOKE_ACCESS_BINDING.operation_id), + prepare_resource=lambda resource: _refresh_artifact_access(request.app.state, {resource.family}), ) return _access_binding_response(binding) @@ -2088,6 +2108,7 @@ async def replace_access_binding( expires_at=payload.replacement.expires_at, ), context=_access_audit_context(REPLACE_ACCESS_BINDING.operation_id), + prepare_resource=lambda resource: _refresh_artifact_access(request.app.state, {resource.family}), ) return TransportAccessBindingReplacement( previous=_access_binding_response(result.previous), @@ -2824,6 +2845,8 @@ async def search_artifacts( if artifacts is None: raise ArtifactSearchUnsupported(family, field="family") access = access_control_for_mode(http_request.app.state.access_control, mode=http_request.app.state.access_mode) + if access is not None: + await _refresh_artifact_access(http_request.app.state, {family}) execution_context = ArtifactSearchExecutionContext( principal=_require_principal() if access is not None else None, access=access, @@ -5010,6 +5033,15 @@ async def authorize(request: Request) -> None: return authorize +async def _refresh_artifact_access(state: Any, families: set[str | None]) -> None: + refresh = getattr(state, "artifact_access_refresher", None) + if refresh is None: + return + missing = {family for family in families if family is not None and family not in ARTIFACT_FAMILY_PROFILES} + if missing: + await refresh(missing) + + async def _authorize_checks( request: Request, access: AccessControlService, @@ -5017,6 +5049,7 @@ async def _authorize_checks( *, operation: str, ) -> None: + await _refresh_artifact_access(request.app.state, {resource.family for _, resource in checks}) context = _access_audit_context(operation) principal = current_principal() decisions = ( diff --git a/src/powercontext/server/artifact_plugins.py b/src/powercontext/server/artifact_plugins.py index 04d2d81341..04fb4f4b19 100644 --- a/src/powercontext/server/artifact_plugins.py +++ b/src/powercontext/server/artifact_plugins.py @@ -65,7 +65,7 @@ EnableArtifactPluginResult, ReloadArtifactPluginRequest, ) -from powercontext.server.authz.profiles import register_custom_artifact_access_family +from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES, register_custom_artifact_access_family _STABLE_PHASES = frozenset({"enabled", "disabled", "unloaded", "failed"}) _STATES = ARTIFACT_PROCESSING_BINDING_STATES_TABLE @@ -82,6 +82,28 @@ logger = logging.getLogger(__name__) +async def read_artifact_declarations(database: AsyncDatabase) -> list[dict[str, Any]]: + async with database.transaction() as connection: + return list( + ( + await connection.execute( + select(_SCHEMAS.c.declaration).order_by(_SCHEMAS.c.family, _SCHEMAS.c.schema_version) + ) + ).scalars() + ) + + +async def refresh_artifact_access_families(database: AsyncDatabase, families: set[str]) -> None: + families = families - ARTIFACT_FAMILY_PROFILES.keys() + if not families: + return + statement = select(_SCHEMAS.c.family).distinct().where(_SCHEMAS.c.family.in_(families)) + async with database.transaction() as connection: + registered = (await connection.execute(statement)).scalars().all() + for family in registered: + register_custom_artifact_access_family(family) + + class ArtifactPluginManager: """Own registrations and transitions; user code stays outside the Server process.""" @@ -212,16 +234,6 @@ async def list_registrations(self) -> list[ArtifactPluginState]: ) return [self._state(dict(row)) for row in rows] - async def declarations(self) -> list[dict[str, Any]]: - async with self.database.transaction() as connection: - return list( - ( - await connection.execute( - select(_SCHEMAS.c.declaration).order_by(_SCHEMAS.c.family, _SCHEMAS.c.schema_version) - ) - ).scalars() - ) - async def _discover(self) -> None: directory = self.plugins_dir if directory is None: diff --git a/src/powercontext/server/authz/service.py b/src/powercontext/server/authz/service.py index 0c96ccad6c..5372f4a88b 100644 --- a/src/powercontext/server/authz/service.py +++ b/src/powercontext/server/authz/service.py @@ -1404,6 +1404,7 @@ async def revoke_binding( expected_version: int, idempotency_key: str, context: AccessAuditContext, + prepare_resource: Callable[[ResourceRef], Awaitable[None]] | None = None, ) -> AccessBinding: _validate_idempotency_key(idempotency_key) actor = _required_principal(principal) @@ -1414,6 +1415,8 @@ async def revoke_binding( if decision.allowed: raise AccessBindingNotFoundError raise AccessDeniedError + if prepare_resource is not None: + await prepare_resource(binding.resource) await self.require_any( actor, _administrative_checks(binding.resource, deployment_id=self.deployment_id), @@ -1444,6 +1447,7 @@ async def replace_binding( request: ReplaceBinding, *, context: AccessAuditContext, + prepare_resource: Callable[[ResourceRef], Awaitable[None]] | None = None, ) -> BindingReplacement: actor = _required_principal(principal) current = await _access_call(self._relationship_reader().get_binding(request.binding_id)) @@ -1457,6 +1461,8 @@ async def replace_binding( if decision.allowed: raise AccessBindingNotFoundError raise AccessDeniedError + if prepare_resource is not None: + await prepare_resource(current.resource) validate_binding_subject(current.resource, current.role, request.subject.type) if isinstance(request.subject, GroupRef) and not self.provider_capabilities.group_subjects: raise AccessInvalidRequestError("group-subjects-unavailable") diff --git a/src/powercontext/server/factory.py b/src/powercontext/server/factory.py index 4f0f53e6f3..8e2cb288f6 100644 --- a/src/powercontext/server/factory.py +++ b/src/powercontext/server/factory.py @@ -20,6 +20,7 @@ import logging from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from contextlib import AsyncExitStack, asynccontextmanager, nullcontext, suppress +from functools import partial from pathlib import Path from fastapi import FastAPI, Request, Response @@ -64,7 +65,11 @@ from powercontext.paths import default_scheduler_path from powercontext.server.access import HttpAccessLogMiddleware from powercontext.server.app import _bind_evidence_access, access_readiness_checks, create_app -from powercontext.server.artifact_plugins import ArtifactPluginManager +from powercontext.server.artifact_plugins import ( + ArtifactPluginManager, + read_artifact_declarations, + refresh_artifact_access_families, +) from powercontext.server.authentication import ( AuthenticationProvider, StaticBearerAuthenticationProvider, @@ -293,6 +298,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: await identity_repository.initialize() server_id = await identity_repository.load_or_create() records = await _restore_artifact_access_profiles(runtime) + app.state.artifact_access_refresher = partial(refresh_artifact_access_families, records.database) _bind_evidence_access(runtime, active_access_control, resolved.access.mode) if ( config.runtime.artifact_processing_supervisor_mode == "global" @@ -348,6 +354,7 @@ def current_capabilities() -> Capabilities: } ) + app.state.artifact_declaration_provider = partial(read_artifact_declarations, records.database) app.state.capabilities = capabilities app.state.capability_provider = current_capabilities app.state.server_info = server_info(server_id) @@ -360,6 +367,8 @@ def current_capabilities() -> Capabilities: _log_lifecycle("server.stopping", "PowerContext Server is stopping") readiness_probe.unbind() app.state.artifact_plugin_manager = None + app.state.artifact_declaration_provider = None + app.state.artifact_access_refresher = None app.state.application = None app.state.dashboard_runtime = None app.state.capability_provider = None diff --git a/tests/e2e/test_artifact_authorization_startup.py b/tests/e2e/test_artifact_authorization_startup.py index acc0a9da56..92c58870c5 100644 --- a/tests/e2e/test_artifact_authorization_startup.py +++ b/tests/e2e/test_artifact_authorization_startup.py @@ -89,7 +89,12 @@ def application(principal): insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( family=family, schema_version="1", - declaration={"family": family, "schema_version": "1"}, + declaration={ + "family": family, + "schema_version": "1", + "content_schema": {"type": "object"}, + "search_fields": [], + }, ) ) for scope in scopes: @@ -114,6 +119,28 @@ def application(principal): ) # Model an empty process-local registry on restart, preserving built-in profiles. monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + discovery_app = application(admin) + async with ( + discovery_app.router.lifespan_context(discovery_app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=discovery_app), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + metadata = await client.get("/v1/capabilities") + assert metadata.status_code == 200, metadata.text + assert family in metadata.json()["artifact_families"] + assert next( + item for item in metadata.json()["custom_artifact_schemas"] if item["family"] == family + ) == { + "family": family, + "schema_version": "1", + "content_schema": {"type": "object"}, + "search_fields": [], + "search": None, + } + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) app = application(reader) async with ( app.router.lifespan_context(app), @@ -135,3 +162,340 @@ def application(principal): assert unknown.status_code in {404, 422}, unknown.text asyncio.run(scenario()) + + +def test_live_family_discovery_and_access_query_budget(tmp_path, monkeypatch): + """External storage writes do not update this Server's process-local profiles.""" + from sqlalchemy import event + + async def scenario(): + async with _isolated_database("sqlite", tmp_path) as database: + settings = ServerSettings( + database=database, + workspace=tmp_path, + runtime=RuntimeConfig(artifact_processing_families=(), artifact_processing_supervisor_mode="dedicated"), + access=AccessControlConfig(mode="enforced"), + dashboard=DashboardConfig(enabled=False), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=True), + ) + await asyncio.to_thread(prepare_server_database, settings, evidence_dir=tmp_path / "evidence") + admin = PrincipalRef(type="service", id="admin") + async with open_builtin_access_control(database, bootstrap_administrators=(admin,)) as access: + app = create_server_app( + settings=settings, + scheduler_path=tmp_path / "scheduler.db", + access_control=access, + authentication_provider=StaticBearerAuthenticationProvider("test-token", admin), + ) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + response = await client.post( + "/v1/scopes", json={"title": "live", "summary": "live", "idempotency_key": "live"} + ) + assert response.status_code == 201, response.text + scope = response.json()["scope_id"] + db = app.state.application._records().database + queries = [] + binding_reads = [] + assert access.relationships is not None + get_binding = access.relationships.get_binding + + async def counted_get_binding(binding_id): + binding_reads.append(binding_id) + return await get_binding(binding_id) + + monkeypatch.setattr(access.relationships, "get_binding", counted_get_binding) + + def track(_conn, _cursor, statement, _parameters, _context, _many): + if ( + statement.lstrip().upper().startswith("SELECT") + and "pc_artifact_family_schemas" in statement + ): + queries.append(statement) + + event.listen(db.engine.sync_engine, "before_cursor_execute", track) + try: + for entry in ( + "read", + "capabilities", + "access/check", + "roles/list", + "bindings/list", + "bindings/create", + "resources/list", + "bindings/revoke", + "bindings/replace", + ): + family = "live-" + entry.replace("/", "-") + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + # Model a different process committing storage without touching this registry. + async with db.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={ + "family": family, + "schema_version": "1", + "content_schema": {}, + "search_fields": [], + }, + ) + ) + await custom_artifact_repository(family).create( + connection, + scope, + "report", + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent(schema_version="1", content={"text": "live"}), + ), + ) + assert family not in ARTIFACT_FAMILY_PROFILES + path = f"/v1/scopes/{scope}/artifacts/{family}/report" + queries.clear() + await _assert_family_display(client, family, present=False) + assert queries == [], "Family display must not refresh the schema registry" + binding_reads.clear() + result = await _request_family_operation(client, access, admin, entry, family, scope, path) + expected_status = ( + 422 if entry in {"bindings/create", "bindings/replace", "resources/list"} else 200 + ) + assert result.status_code == expected_status, result.text + assert len(queries) == (2 if entry == "read" else 1), queries + _assert_discovered_family(entry, family, result.json()) + if entry in {"bindings/revoke", "bindings/replace"}: + assert binding_reads == [family], "Binding ID must be resolved only once" + if entry in { + "roles/list", + "bindings/list", + "bindings/create", + "resources/list", + "bindings/replace", + }: + queries.clear() + repeated = await _request_family_operation( + client, access, admin, entry, family, scope, path + ) + assert repeated.status_code == result.status_code, repeated.text + assert repeated.json() == result.json() + assert queries == [], "Known family management must not refresh the registry" + queries.clear() + await _assert_family_display(client, family, present=True) + assert queries == [], "Family display must use the local registry" + for suffix in ("", "/revisions/1"): + read = await client.get(path + suffix) + assert read.status_code == 200, read.text + assert len(queries) == 2, queries # Existing persisted-family validation: one per read. + finally: + event.remove(db.engine.sync_engine, "before_cursor_execute", track) + + asyncio.run(scenario()) + + +def _assert_discovered_family(entry, family, payload): + if entry == "capabilities": + assert family in payload["artifact_families"] + assert any(item["family"] == family for item in payload["custom_artifact_schemas"]) + elif entry == "access/check": + assert payload["allowed"] is True + + +async def _assert_family_display(client, family, *, present): + response = await client.get("/v1/access/me") + assert response.status_code == 200, response.text + assert any(item["family"] == family for item in response.json()["artifact_families"]) is present + response = await client.get("/health/ready") + assert response.status_code == 200, response.text + assert (f"{family}:enabled" in response.json()["checks"]["access_artifact_families"]) is present + + +async def _request_family_operation(client, access, admin, entry, family, scope, path): + if entry == "read": + return await client.get(path) + if entry == "capabilities": + return await client.get("/v1/capabilities") + resource = {"type": "artifact", "scope_id": scope, "identity": {"family": family, "artifact_id": "report"}} + subject = {"type": "user", "id": "recipient"} + payloads = { + "access/check": {"requirements": [{"action": "artifact.read", "resource": resource}], "match": "all"}, + "roles/list": {"resource_type": "artifact", "family": family}, + "bindings/list": {"management_resource": resource}, + "bindings/create": { + "resource": resource, + "subject": subject, + "role": "artifact.viewer", + "idempotency_key": family, + }, + "resources/list": {"resource_type": "artifact", "family": family, "action": "artifact.read"}, + } + if entry in {"bindings/revoke", "bindings/replace"}: + await _seed_retained_binding(access, admin, family, scope) + payload = {"binding_id": family, "expected_version": 1, "idempotency_key": family + "-change"} + if entry == "bindings/replace": + payload["replacement"] = {"subject": subject} + else: + payload = payloads[entry] + endpoint = entry if entry == "access/check" else "access/" + entry + return await client.post("/v1/" + endpoint, json=payload) + + +async def _seed_retained_binding(access, admin, family, scope): + from datetime import UTC, datetime + + from powercontext.server.authz.models import AccessBinding, AccessBindingState + + # Exercise an existing relationship row; custom families still cannot grant direct roles through HTTP. + await access.relationships.create_binding( + AccessBinding( + binding_id=family, + subject=admin, + resource=ResourceRef.artifact(scope, family=family, artifact_id="report"), + role=AccessRole.ARTIFACT_OWNER, + granted_by=admin, + reason=None, + created_at=datetime.now(UTC), + expires_at=None, + state=AccessBindingState.ACTIVE, + version=1, + policy_revision="pending", + idempotency_key=family, + ) + ) + + +class _FilteringProvider: + """A custom Provider that retains Scope grants but denies one exact Artifact.""" + + def __init__(self, delegate): + self.delegate = delegate + + async def check(self, request): + from dataclasses import replace + + decision = await self.delegate.check(request) + if request.resource.identity is not None and request.resource.identity.artifact_id == "hidden": + return replace( + decision, allowed=False, reason_code="no-matching-policy", matched_subject=None, matched_binding_id=None + ) + return decision + + async def check_batch(self, requests): + return tuple([await self.check(request) for request in requests]) + + async def resolve_resource_filter(self, request): + return await self.delegate.resolve_resource_filter(request) + + +def test_first_search_loads_external_family_with_custom_provider(tmp_path, monkeypatch): + from sqlalchemy import event + + from powercontext.artifacts.indexes import ArtifactSearchDeclaration, TextIndex + from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder + + async def scenario(): + family = "live-search-report" + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + async with _isolated_database("sqlite", tmp_path) as database: + settings = ServerSettings( + database=database, + workspace=tmp_path, + access=AccessControlConfig(mode="enforced"), + runtime=RuntimeConfig(artifact_processing_families=(), artifact_processing_supervisor_mode="dedicated"), + dashboard=DashboardConfig(enabled=False), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings, evidence_dir=tmp_path / "evidence") + admin = PrincipalRef(type="service", id="admin") + async with open_builtin_access_control(database, bootstrap_administrators=(admin,)) as access: + access.provider = _FilteringProvider(access.provider) + app = create_server_app( + settings=settings, + scheduler_path=tmp_path / "scheduler.db", + access_control=access, + authentication_provider=StaticBearerAuthenticationProvider("test-token", admin), + ) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + response = await client.post( + "/v1/scopes", json={"title": "search", "summary": "search", "idempotency_key": "search"} + ) + assert response.status_code == 201, response.text + scope = response.json()["scope_id"] + db = app.state.application._records().database + search = ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="text", fields=("text",), projection_version="1"),), + modes=("text",), + default_mode="text", + ) + # Commit external storage changes without touching this Server's profile registry. + async with db.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={ + "family": family, + "schema_version": "1", + "content_schema": {}, + "search_fields": [], + "search": search.model_dump(mode="json"), + }, + ) + ) + for identity in ("visible", "hidden"): + await custom_artifact_repository(family).create( + connection, + scope, + identity, + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent( + schema_version="1", content={"text": "database timeout"} + ), + ), + ) + builder = ArtifactIndexBuilder(db) + await builder.refresh(scope, family) + assert await builder.build(scope, family) + assert family not in ARTIFACT_FAMILY_PROFILES + queries = [] + + def track(_conn, _cursor, statement, _parameters, _context, _many): + if ( + statement.lstrip().upper().startswith("SELECT") + and "pc_artifact_family_schemas" in statement + ): + queries.append(statement) + + event.listen(db.engine.sync_engine, "before_cursor_execute", track) + try: + path = f"/v1/scopes/{scope}/artifacts/{family}/search" + first = await client.post(path, json={"query": "timeout"}) + assert first.status_code == 200, first.text + assert [item["artifact_id"] for item in first.json()["results"]] == ["visible"] + first_queries = len(queries) + queries.clear() + second = await client.post(path, json={"query": "timeout"}) + assert second.status_code == 200, second.text + assert second.json() == first.json() + assert len(queries) == first_queries - 1, "Only the first search may load the missing profile" + finally: + event.remove(db.engine.sync_engine, "before_cursor_execute", track) + + asyncio.run(scenario()) From 0d78bcb14d51cda1b29d1adacce13568ad2e8cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Fri, 30 Oct 2026 10:35:10 +0800 Subject: [PATCH 08/10] fix(artifact): preserve worker inference configuration --- .../builtin/runtime/artifact_plugins.py | 8 +- .../runtime/test_artifact_plugin_worker.py | 119 ++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 tests/builtin/runtime/test_artifact_plugin_worker.py diff --git a/src/powercontext/builtin/runtime/artifact_plugins.py b/src/powercontext/builtin/runtime/artifact_plugins.py index 1eb3123331..db91b2f59e 100644 --- a/src/powercontext/builtin/runtime/artifact_plugins.py +++ b/src/powercontext/builtin/runtime/artifact_plugins.py @@ -56,7 +56,7 @@ ARTIFACT_PROCESSING_BINDING_STATES_TABLE, ) from powercontext.builtin.runtime.artifact_reads import ArtifactReadSession -from powercontext.builtin.runtime.config import BuiltinConfig, DatabaseConfig, RuntimeConfig +from powercontext.builtin.runtime.config import BuiltinConfig, DatabaseConfig, InferenceConfig, RuntimeConfig from powercontext.builtin.runtime.processing_contracts import ( ArtifactProcessingWorkAssignment, ArtifactProcessingWorkerCompletion, @@ -132,7 +132,11 @@ async def _process_window(spec: ArtifactPluginWorkerSpec, assignment: ArtifactPr embedding = None if spec.inference_config is not None: embedding, _ = await _embedding_models(spec.inference_config, resources, None) - config = BuiltinConfig(database=spec.database, runtime=spec.runtime_config or RuntimeConfig()) + config = BuiltinConfig( + database=spec.database, + inference=spec.inference_config or InferenceConfig(), + runtime=spec.runtime_config or RuntimeConfig(), + ) storage = await resources.enter_async_context( open_builtin_contexts(config, embedding_model=embedding, _topic_memory_worker=True) ) diff --git a/tests/builtin/runtime/test_artifact_plugin_worker.py b/tests/builtin/runtime/test_artifact_plugin_worker.py new file mode 100644 index 0000000000..ad559cac4b --- /dev/null +++ b/tests/builtin/runtime/test_artifact_plugin_worker.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Plugin Workers preserve deployment inference identity when opening storage.""" + +import asyncio +import hashlib +import json + +import pytest +import yaml +from sqlalchemy import insert + +from powercontext.builtin.artifact_plugin_validation import _PluginManifest, describe_module, load_plugin +from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository +from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, +) +from powercontext.builtin.runtime.artifact_plugins import ArtifactPluginWorkerSpec, run_artifact_plugin_worker +from powercontext.builtin.runtime.composition import open_builtin_contexts +from powercontext.builtin.runtime.config import BuiltinConfig, InferenceConfig, SQLiteConfig +from powercontext.builtin.runtime.processing_contracts import ArtifactProcessingWorkAssignment +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, ContentSource +from powercontext.sources import SourceMaterialization +from tests.e2e.test_artifact_plugins import EXAMPLE, deploy_plugin + + +@pytest.mark.parametrize("configured_inference", [False, True]) +def test_plugin_worker_generates_with_deployment_inference(tmp_path, configured_inference): + directory = str(deploy_plugin(EXAMPLE, tmp_path / "plugin", None)) + manifest = _PluginManifest.model_validate(yaml.safe_load((tmp_path / "plugin/plugin.yml").read_text())) + definition = describe_module(load_plugin(directory, manifest), manifest, directory) + definition["directory"] = directory + declaration = definition["declaration"] + family, binding = declaration["family"], manifest.binding_name + # Only storage composition is used: this model is never called by the deterministic plugin. + inference = InferenceConfig(generation_model="openai:gpt-4o-mini") if configured_inference else None + config = BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'worker.db'}"), + inference=inference or InferenceConfig(), + ) + + async def prepare(): + async with open_builtin_contexts(config) as storage, storage.database.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration=declaration, + ) + ) + await connection.execute( + insert(ARTIFACT_PROCESSING_BINDING_STATES_TABLE).values( + binding_name=binding, + binding_generation=1, + phase="enabled", + ) + ) + source = await SourceRepository(BUILTIN_SOURCE_REGISTRY).add( + connection, + "scope", + ContentSource( + name="ticket", + materialization=SourceMaterialization.CAPTURED, + content="Example ticket", + ), + ) + lease = await ArtifactProcessingLeaseRepository().start_single_process_term(connection, "test-owner") + intent = await ArtifactProcessingIntentRepository().request(connection, "scope", binding) + assignment = ArtifactProcessingWorkAssignment( + binding_name=binding, + scope_id="scope", + artifact_family=family, + claimed_request_generation=intent.requested_generation, + fence=lease.fence("single-process"), + worker_id="test-worker", + ) + return assignment, source.ref + + assignment, source_ref = asyncio.run(prepare()) + run_artifact_plugin_worker( + ArtifactPluginWorkerSpec( + config.database, + binding, + 1, + definition, + inference_config=inference, + runtime_config=config.runtime, + ), + assignment, + ) + + async def verify(): + async with open_builtin_contexts(config) as storage, storage.database.transaction() as connection: + identity = json.dumps(source_ref.model_dump(mode="json"), sort_keys=True, separators=(",", ":")) + artifact_id = hashlib.sha256(identity.encode()).hexdigest() + report = await custom_artifact_repository(family).latest(connection, "scope", family, artifact_id) + assert report.content.content["evidence_text"] == "Example ticket" + assert report.lineage.sources == (source_ref,) + intent = await ArtifactProcessingIntentRepository().load(connection, "scope", binding) + assert intent is not None + assert intent.handled_generation == assignment.claimed_request_generation + + asyncio.run(verify()) From 6d567f4e55c7324174dc375e01bb31269d5161d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Sat, 31 Oct 2026 10:47:13 +0800 Subject: [PATCH 09/10] fix: preserve core-only package imports --- .github/workflows/master.yml | 8 ++++++-- scripts/ci_role_smoke.py | 17 +++++++++++++++-- src/powercontext/__init__.py | 17 ++++++++++++++++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index a661ba42aa..f37d12de6c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -254,7 +254,7 @@ jobs: fail-fast: false matrix: python-version: ["3.11", "3.14"] - extras: ["builtin", "cli", "builtin,cli", "server,cli"] + extras: ["core", "builtin", "cli", "builtin,cli", "server,cli"] steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -278,7 +278,11 @@ jobs: uv build --wheel --out-dir "$RUNNER_TEMP/role-wheel" uv venv --python "$(command -v python)" "$RUNNER_TEMP/role-venv" wheel="$(find "$RUNNER_TEMP/role-wheel" -name '*.whl' -print -quit)" - uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel[$ROLE_EXTRAS]" + if [ "$ROLE_EXTRAS" = core ]; then + uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel" + else + uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel[$ROLE_EXTRAS]" + fi - name: Verify installed role behavior without checkout imports env: diff --git a/scripts/ci_role_smoke.py b/scripts/ci_role_smoke.py index c18980606c..9c79cd3461 100644 --- a/scripts/ci_role_smoke.py +++ b/scripts/ci_role_smoke.py @@ -18,6 +18,7 @@ import argparse import asyncio +import importlib.util import os import subprocess import sys @@ -99,8 +100,20 @@ def _check_cli(root: Path, extras: set[str]) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("extras", choices=("builtin", "cli", "builtin,cli", "server,cli")) - extras = set(parser.parse_args().extras.split(",")) + parser.add_argument("extras", choices=("core", "builtin", "cli", "builtin,cli", "server,cli")) + role = parser.parse_args().extras + extras = set() if role == "core" else set(role.split(",")) + import powercontext + + if powercontext.PowerContext is None or powercontext.ArtifactRef is None: + raise RuntimeError("Core public exports are unavailable") # noqa: TRY003 + if not extras and importlib.util.find_spec("httpx") is not None: + raise RuntimeError("Core installation unexpectedly includes httpx") # noqa: TRY003 + if extras: + from powercontext import artifact_plugins + + if powercontext.artifact_plugins is not artifact_plugins or artifact_plugins.ArtifactFamily is None: + raise RuntimeError("Plugin public export is unavailable") # noqa: TRY003 if "cli" in extras: extras.add("builtin") with tempfile.TemporaryDirectory(prefix="powercontext-role-") as temporary: diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index f6d9156e1f..65914fe458 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -14,7 +14,10 @@ """Stable public exports for PowerContext clients.""" -from powercontext import artifact_plugins +from importlib import import_module +from types import ModuleType +from typing import TYPE_CHECKING + from powercontext.artifacts import ( Artifact, ArtifactAddress, @@ -116,3 +119,15 @@ "Trigger", "artifact_plugins", ] + + +if TYPE_CHECKING: + from powercontext import artifact_plugins + + +def __getattr__(name: str) -> ModuleType: + if name == "artifact_plugins": + module = import_module("powercontext.artifact_plugins") + globals()[name] = module + return module + raise AttributeError(name) From 322b7a956ae58230b9305cd943ce66a7d23ba868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=98=E9=9C=96?= Date: Sat, 31 Oct 2026 11:03:45 +0800 Subject: [PATCH 10/10] test: cover source to artifact plugin flow --- tests/e2e/test_source_artifact_plugins.py | 91 +++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tests/e2e/test_source_artifact_plugins.py diff --git a/tests/e2e/test_source_artifact_plugins.py b/tests/e2e/test_source_artifact_plugins.py new file mode 100644 index 0000000000..2a29f35f0c --- /dev/null +++ b/tests/e2e/test_source_artifact_plugins.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Source plugin snapshot flows through a real Artifact Worker to searchable evidence.""" + +import asyncio +import shutil + +from fastmcp import Client + +from tests.e2e.test_artifact_plugins import EXAMPLE, artifacts, change, deploy_plugin, reload_plugin +from tests.source_helpers import ROOT, TOKENS, call, create_scope, grant, load_write_cli, read, running_server + + +def test_source_plugin_to_artifact_search_and_exact_read(tmp_path): + asyncio.run(_source_to_artifact(tmp_path)) + + +async def _source_to_artifact(directory): + source_plugin = directory / "tickets" + shutil.copytree( + ROOT / "examples/source_plugins/tickets", source_plugin, ignore=shutil.ignore_patterns("__pycache__") + ) + report_plugin = deploy_plugin(EXAMPLE, directory / "reports", None) + entry = report_plugin / "plugin.py" + # Use the delivered generator with the Source plugin's persisted text projection. + entry.write_text( + entry.read_text().replace( + '"content": TEXT_EVIDENCE_PROJECTION_KEY', '"example-ticket": TEXT_EVIDENCE_PROJECTION_KEY' + ) + ) + value = { + "namespace": "support", + "project": "powercontext", + "ticket_id": "PC-123", + "title": "数据库恢复", + "body": "Outage resolved after database retry.", + } + environment = {"POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR": str(directory / "plugins")} + (directory / "plugins").mkdir() + async with running_server(directory, environment=environment) as (admin, url, _process): + scope = await create_scope(admin, "source-artifact-chain") + await grant(admin, scope, "deployer") + await grant(admin, scope) + await load_write_cli(source_plugin / "write.yml", url) + await reload_plugin(admin, directory, report_plugin, 0) + await change(admin, "enable", 1, "enabled") + async with Client(url + "/mcp", auth=TOKENS["host"]) as host: + captured = await call( + host, + "pc_source_add", + { + "scope_id": scope, + "capability": "ticket.write", + "capability_version": "1", + "input": value, + }, + ) + source_ref = captured["result"]["source_refs"][0] + assert source_ref["source_type"] == "example-ticket" + assert (await read(admin, scope, source_ref))["content"]["payload"]["capture"]["body"] == value["body"] + [artifact] = await artifacts(admin, scope, 1) + async with asyncio.timeout(30): + while True: + response = await admin.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": "数据库恢复"} + ) + if response.status_code == 200 and response.json()["results"]: + break + assert response.status_code in (200, 503), response.text + await asyncio.sleep(0.1) + [hit] = response.json()["results"] + assert hit["artifact_id"] == artifact["artifact_id"] + assert hit["revision"] == artifact["revision"] + exact = await admin.get( + f"/v1/scopes/{scope}/artifacts/incident-report/{hit['artifact_id']}/revisions/{hit['revision']}" + ) + assert exact.status_code == 200, exact.text + assert exact.json()["content"]["evidence_text"] == value["title"] + "\n\n" + value["body"] + assert exact.json()["sources"] == [source_ref]