diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index a661ba42aa..f37d12de6c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -254,7 +254,7 @@ jobs: fail-fast: false matrix: python-version: ["3.11", "3.14"] - extras: ["builtin", "cli", "builtin,cli", "server,cli"] + extras: ["core", "builtin", "cli", "builtin,cli", "server,cli"] steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -278,7 +278,11 @@ jobs: uv build --wheel --out-dir "$RUNNER_TEMP/role-wheel" uv venv --python "$(command -v python)" "$RUNNER_TEMP/role-venv" wheel="$(find "$RUNNER_TEMP/role-wheel" -name '*.whl' -print -quit)" - uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel[$ROLE_EXTRAS]" + if [ "$ROLE_EXTRAS" = core ]; then + uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel" + else + uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel[$ROLE_EXTRAS]" + fi - name: Verify installed role behavior without checkout imports env: diff --git a/desktop/VALIDATION.md b/desktop/VALIDATION.md index c684257b2d..4c479b5268 100644 --- a/desktop/VALIDATION.md +++ b/desktop/VALIDATION.md @@ -51,7 +51,7 @@ The historical Server c535d816 protocol qualification below records the specific ## Compatibility scope -[compatibility.json](src-tauri/src/connections/compatibility.json) preserves each recorded Server source, normalized OpenAPI contract digest, qualification-wheel digest and operation set. The bundled contract matches `sqlite-atomic-0a49db9b-v1`, which qualifies the eleven protocol operations recorded below. Existing records retain their exact source and contract identities. The `sqlite-atomic-dd62723d-v1` and `sqlite-atomic-4ffdf4e9-v1` records retain their migration-framework and release-preparation source qualifications with their original contract digest. Other historical records also retain their exact identities and evidence. Ordinary builds require explicitly selecting the record whose tested Server source and wheel match the deployment, then checking the connection before business operations become available. Records with a different contract digest are disabled in the selector, and saved unavailable selections remain visible until explicitly replaced. Selecting a profile does not extend its recorded operation set or qualify a different Server source. CI builds have their own artifact identities and do not change product qualification. +[compatibility.json](src-tauri/src/connections/compatibility.json) preserves each recorded Server source, normalized OpenAPI contract digest, qualification-wheel digest and operation set. `sqlite-atomic-cc5980f0-v1` records the ten protocol operations qualified below. The bundled contract is qualified by `sqlite-source-artifact-87b6fae2-v1` for the exact source snapshot and wheel recorded below. The preceding `sqlite-artifact-merge-87d01f68-v1` record retains its original qualification. The `sqlite-atomic-dd62723d-v1` and `sqlite-atomic-4ffdf4e9-v1` records retain their migration-framework and release-preparation source qualifications with their original contract digest. Other historical records also retain their exact identities and evidence. Ordinary builds require explicitly selecting the record whose tested Server source and wheel match the deployment, then checking the connection before business operations become available. Records with a different contract digest are disabled in the selector, and saved unavailable selections remain visible until explicitly replaced. Selecting a profile does not extend its recorded operation set or qualify a different Server source. CI builds have their own artifact identities and do not change product qualification. The real Server harness covers SQLite with anonymous loopback, static Bearer, HTTPS with explicit CA/base path, and injected-provider/enforced access. It exercises save/search/exact reads, identity changes, binding revocation, ambiguous writes without replay and Scope pagination. Installed UI checks cover repeated launches preserving the existing editor, late committed-save responses preserving a new Scope draft, explicit connection/Scope selection, clipboard copies, connection isolation, draft cancellation, byte limits and independent Server survival after Desktop exit. @@ -189,6 +189,69 @@ original source, contract and wheel identities. 原生探针通过匿名回环、Bearer、显式 CA／路径前缀 HTTPS 及强制访问控制四种真实 SQLite 协议验收。该记录仅覆盖所列十项操作，不包含 manifest 摘要、Windows、安装包或原生凭据库验收。 +#### Artifact framework merge 87d01f68 protocol qualification + +`sqlite-artifact-merge-87d01f68-v1` identifies the uncommitted merge of branch HEAD +`0d78bcb14d51cda1b29d1adacce13568ad2e8cd4` and mainline +`90355b8e96d68e2932d7980b3e76957f35ea2b44`, including the migration compatibility adjustments. +Its `serverCommit` is a working-tree identity, not a Git commit. The source snapshot SHA-256 is +`87d01f6868f8337840f57985fd9be7f0772120d845d1b4ae80895e15724a5a80`: SHA-256 of sorted `path sha256` lines, each ending in a newline, +for all packaged runtime files at their `src/` paths plus `pyproject.toml`, `uv.lock` +and `openapi/powercontext.yaml`. All 638 packaged runtime files match the source byte for byte. + +OpenAPI SHA-256: `aa46c5f141190bc3516a34da9fd07ce756bdcf7537ab7109f151755e28cc6ed7`. +Wheel: `powercontext-1.4.1.dev235+g0d78bcb14.d20261030-py3-none-any.whl`. +Wheel SHA-256: `4806c56e3e948249ef097e504e11347833b1d8fe510f4b7639ef3eca16460ce2`. +The development version is build metadata and does not change the formal release version. + +On 2026-10-30, Linux x86_64, Python 3.12.12 and Rust 1.95.0 passed all four +built-wheel SQLite modes in `desktop/tests/real_server.py`: anonymous loopback, Bearer, +HTTPS with an explicit CA/base path, and injected-provider/enforced access. The unchanged +`server_probe.rs` covers save/search/exact reads, explicit activation and Scope selection, +committed writes with lost responses without replay, identity changes, revocation, Scope +pagination and Server survival after client exit. The record qualifies the same ten operations +as the preceding Artifact profile; plugin administration APIs are outside this qualification. + +The 13 native source/probe files were copied unchanged into an isolated crate without +Tauri/UI dependencies. The local build used vendored DBus because the host lacked DBus +development headers. A local harness wrapper retained the 180-second fixture startup +allowance; protocol assertions and product timeouts were unchanged. This establishes SQLite +protocol compatibility for the recorded snapshot and wheel, not native credential-store, +installed Windows/macOS UI, signing or release acceptance. Historical records remain unchanged. + +#### Artifact migration merge protocol qualification + +`sqlite-artifact-merge-e30479df-v1` qualifies OpenAPI digest +`bf9c258adaa03a652093fbc83f449a5eff95ac6e018cc7281b5c530a2214f7a9` and wheel +`powercontext-1.4.1.dev179+g64b5f8e42.d20261027-py3-none-any.whl`, SHA-256 +`6daf419858ae4eb618d49bc5e0cc3f3cc47d7d8ef473c8801b27d849b7c7b406`. + +The tested source is the uncommitted merge of branch HEAD +`64b5f8e426019c4bd242a49818eb52feec9d32b7` and mainline +`8ef4f645e66622e223cd49720d5f009f63ecff85`, including Artifact migration compatibility changes. +The record's `serverCommit` value explicitly identifies a working-tree snapshot, not a Git commit. +Its source snapshot SHA-256 is `e30479dfd8e034b1cdca8a9b59d8b9860b350d9f1acc1f57e1b20a4f3d13c7b0`. +This hashes sorted `path sha256` lines, each ending in a newline, for all packaged runtime files +at their `src/` paths plus `pyproject.toml`, `uv.lock` and `openapi/powercontext.yaml`. +All 625 packaged runtime files match the source byte for byte; the corresponding +sorted `wheel/path sha256` manifest digest is `1fdc52ce259767cf0e362e85d43735ed0dad80aac3a9f54dd5b48a11ab235593`. +The wheel's development version is build metadata, not a formal release version change. + +On 2026-10-27, Linux x86_64, Python 3.12.12 and Rust 1.95.0 passed all four +built-wheel SQLite modes in `desktop/tests/real_server.py`: anonymous loopback, Bearer, +HTTPS with an explicit CA/base path, and injected-provider/enforced access. +The unchanged `server_probe.rs` assertions cover save/search/exact reads, committed writes +with lost responses without replay, identity changes, revocation, Scope pagination and +Server survival after client exit. The record grants the same ten Desktop operations as +the preceding Artifact profile; plugin administration APIs are not qualified. + +The unmodified connection, credential, error and transport modules and probe were rebuilt +in an isolated native crate without Tauri/UI dependencies. The initial default 20-second +fixture startup wait expired before protocol assertions. A local harness wrapper allowed +180 seconds for Server startup; protocol assertions and product timeouts were unchanged. +This establishes native SQLite protocol compatibility for the exact source snapshot and +wheel, not installed Windows/macOS UI, native credential-store or release acceptance. + #### Server 68491568 protocol qualification `sqlite-atomic-68491568-v1` qualifies Server sources at `68491568359fd9f5723208bbeeec2f808fd2bcc9` @@ -868,3 +931,42 @@ are documented in [Local Server Ops](../docs/en/docs/operate/local-server-ops.md 上述两项可选检查分别验证源码 Ops 子进程和完整独立分发包与原生桥接的只读兼容性。测试使用临时 home、绑定和状态目录， 不执行服务修改；真实服务管理器归属无法确认时保留预检拒绝。分发包测试要求另行提供已批准的清单摘要，并检查依赖被改动时 原生端拒绝启动。这些证据不代替签名／公证、真实服务修改、随包配置或干净机器验收。 + +## Server d6820d88 protocol qualification + +`sqlite-artifact-d6820d88-v1` identifies Server sources at `d6820d884ccd240712df477e459622f60fd6f5f5`, contract digest `f571a95d193f0dae91694ffcd6b37c93603a5ae731fddefbcdbfa9a5914762ae`, and wheel digest `5bcedfde3efa4ae028318a9b8848dec44b5604ac6f636f32bacb637ca5729f52`. The wheel `powercontext-0.1.dev0+gd6820d88-py3-none-any.whl` was built from a clean Git archive of that commit with an explicit archive-build version; all 563 runtime files match the archived sources byte for byte. + +On 2026-10-26, the native `ConnectionManager` and HTTP adapter passed all four built-wheel SQLite fixture modes: anonymous loopback, Bearer, HTTPS with an explicit CA/base path, and injected-provider/enforced access. The unchanged `server_probe.rs` assertions cover writes, search, exact historical reads, lost-response writes without replay, identity changes, revocation, and Scope pagination. The qualification records the same ten Desktop operations as the preceding profile; it does not qualify the new plugin administration operations. + +Validation used Linux, Python 3.12.12 and Rust 1.95.0. The unchanged connection, credential and transport modules and probe were built in an isolated native crate without Tauri/UI dependencies. On the loaded host, importing the Server took about 60 seconds, so the local fixture preparation allowance was 180 seconds instead of 20; protocol assertions and product timeouts were unchanged. This establishes native SQLite protocol compatibility, not full Tauri, installed-package, Windows/macOS service, vault or UI acceptance. + +#### Source and Artifact merge 87b6fae2 protocol qualification + +`sqlite-source-artifact-87b6fae2-v1` identifies the uncommitted merge of Artifact HEAD +`079de1030ec0404ad4bba5f31d2d979078a55459` and mainline +`0c1dfc289b28e9f529555fb57bf771ce51103913`, including migration compatibility adjustments. +Its `serverCommit` is a working-tree identity, not a Git commit. Source snapshot SHA-256: +`87b6fae2104aa599f626afff7647be37711151bbc8f8329257c4cab532516f55`. +This hashes sorted `path sha256` lines, each ending in a newline, for all packaged runtime +files at their `src/` paths plus `pyproject.toml`, `uv.lock` and `openapi/powercontext.yaml`. +All 663 packaged runtime files match the source byte for byte. + +OpenAPI SHA-256: `37936d7897ec2d981dcc195d3b7b7a9f99c572d866c0193b37df43617077c155`. +Wheel: `powercontext-1.4.1.dev273+g079de1030.d20261031-py3-none-any.whl`. +Wheel SHA-256: `9915db1a7341622d8f22d97b1c02aa4cbd50a7c3a3be3dc9d9346cb77b3da56f`. +The development version is build metadata; the formal release version is unchanged. + +On 2026-10-31, Linux x86_64, Python 3.12.12 and Rust 1.95.0 passed all four built-wheel +SQLite modes in `desktop/tests/real_server.py`: anonymous loopback, Bearer, HTTPS with an +explicit CA/base path, and injected-provider/enforced access. The existing native probe +covers the eleven declared protocol operations, including server identity, save/search/exact +reads, Scope selection, identity changes and revocation, lost responses without replay, +and Server survival after client exit. Plugin management is outside this qualification. + +The product native protocol modules and probe were copied unchanged into an isolated crate +without the Tauri GUI. The local build used vendored DBus because the host lacked development +headers, and the CI fixture profile was supplied through `OUT_DIR`. The local harness wrapper +retained the 180-second fixture startup allowance; protocol assertions and product timeouts +were unchanged. This qualifies SQLite protocol behavior for this snapshot and wheel, not +Windows/macOS execution, native credential persistence, installed UI or release readiness. +Historical qualification records retain their identities and evidence. diff --git a/desktop/scripts/generate.mjs b/desktop/scripts/generate.mjs index 2a23ff8c8e..b618c187f3 100644 --- a/desktop/scripts/generate.mjs +++ b/desktop/scripts/generate.mjs @@ -91,6 +91,13 @@ function collectSchemas(value) { for (const item of Object.values(contract.paths)) for (const op of Object.values(item)) if (wanted.includes(op?.operationId)) collectSchemas(op); +// OpenAPI Generator inlines plain string aliases instead of emitting model files. +const stringAliases = new Set( + [...schemaNames].filter((name) => { + const schema = contract.components.schemas[name]; + return schema.type === "string" && !schema.enum; + }), +); function snakeCase(name) { return name .replace(/([a-z0-9])([A-Z])/g, "$1_$2") @@ -124,7 +131,9 @@ async function generateRustModels() { join(tmpdir(), "powercontext-desktop-openapi-"), ); const output = join(temporary, "generated"); - const modelNames = [...schemaNames].sort(); + const modelNames = [...schemaNames] + .filter((name) => !stringAliases.has(name)) + .sort(); const generatorCli = createRequire(import.meta.url).resolve( "@openapitools/openapi-generator-cli/main.js", ); @@ -238,7 +247,11 @@ const rustDeclarations = `\n#[rustfmt::skip]\npub fn declarations(config: &ts_rs ...new Set([...schemaNames, ...rustModelNames]), ] .sort() - .map((name) => ` <${name} as ts_rs::TS>::decl(config),`) + .map((name) => + stringAliases.has(name) + ? ` format!("type ${name} = {};", ::name(config)),` + : ` <${name} as ts_rs::TS>::decl(config),`, + ) .join("\n")}\n ]\n}\n`; const outputs = { "src-tauri/src/transport/wire.rs": diff --git a/desktop/src-tauri/src/connections/compatibility.json b/desktop/src-tauri/src/connections/compatibility.json index 5c0f8605d9..8c3417dd59 100644 --- a/desktop/src-tauri/src/connections/compatibility.json +++ b/desktop/src-tauri/src/connections/compatibility.json @@ -305,6 +305,63 @@ ], "evidence": "desktop/VALIDATION.md#server-c535d816-protocol-qualification" }, + { + "id": "sqlite-artifact-d6820d88-v1", + "serverCommit": "d6820d884ccd240712df477e459622f60fd6f5f5", + "contractSha256": "f571a95d193f0dae91694ffcd6b37c93603a5ae731fddefbcdbfa9a5914762ae", + "artifactSha256": "5bcedfde3efa4ae028318a9b8848dec44b5604ac6f636f32bacb637ca5729f52", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#server-d6820d88-protocol-qualification" + }, + { + "id": "sqlite-artifact-merge-e30479df-v1", + "serverCommit": "working-tree-sha256:e30479dfd8e034b1cdca8a9b59d8b9860b350d9f1acc1f57e1b20a4f3d13c7b0", + "contractSha256": "bf9c258adaa03a652093fbc83f449a5eff95ac6e018cc7281b5c530a2214f7a9", + "artifactSha256": "6daf419858ae4eb618d49bc5e0cc3f3cc47d7d8ef473c8801b27d849b7c7b406", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#artifact-migration-merge-protocol-qualification" + }, + { + "id": "sqlite-artifact-merge-87d01f68-v1", + "serverCommit": "working-tree-sha256:87d01f6868f8337840f57985fd9be7f0772120d845d1b4ae80895e15724a5a80", + "contractSha256": "aa46c5f141190bc3516a34da9fd07ce756bdcf7537ab7109f151755e28cc6ed7", + "artifactSha256": "4806c56e3e948249ef097e504e11347833b1d8fe510f4b7639ef3eca16460ce2", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#artifact-framework-merge-87d01f68-protocol-qualification" + }, { "id": "sqlite-atomic-68491568-v1", "serverCommit": "68491568359fd9f5723208bbeeec2f808fd2bcc9", @@ -343,5 +400,25 @@ "search_memory" ], "evidence": "desktop/VALIDATION.md#server-0a49db9b-protocol-qualification" + }, + { + "id": "sqlite-source-artifact-87b6fae2-v1", + "serverCommit": "working-tree-sha256:87b6fae2104aa599f626afff7647be37711151bbc8f8329257c4cab532516f55", + "contractSha256": "37936d7897ec2d981dcc195d3b7b7a9f99c572d866c0193b37df43617077c155", + "artifactSha256": "9915db1a7341622d8f22d97b1c02aa4cbd50a7c3a3be3dc9d9346cb77b3da56f", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "get_server_info", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#source-and-artifact-merge-87b6fae2-protocol-qualification" } ] diff --git a/desktop/src-tauri/src/transport/api.rs b/desktop/src-tauri/src/transport/api.rs index 1c0a692434..ad74a786e4 100644 --- a/desktop/src-tauri/src/transport/api.rs +++ b/desktop/src-tauri/src/transport/api.rs @@ -509,7 +509,7 @@ impl ServerApi { ) .await?; if result.scope_id != scope - || result.family != ArtifactReadFamily::AtomicMemory + || result.family != "atomic-memory" || result.artifact_id != artifact.artifact_id || result.revision != artifact.revision || result.content.get("schema").and_then(|v| v.as_str()) diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 9427a5963d..2b1335db60 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "bce4e0d3deec56590a29c3c06af4d853fc8cad2917d8832b23c92dd160472ccd", + "contractSha256": "37936d7897ec2d981dcc195d3b7b7a9f99c572d866c0193b37df43617077c155", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/src-tauri/src/transport/wire.rs b/desktop/src-tauri/src/transport/wire.rs index d15439ad89..80674e903e 100644 --- a/desktop/src-tauri/src/transport/wire.rs +++ b/desktop/src-tauri/src/transport/wire.rs @@ -369,44 +369,6 @@ pub enum ShareUnit { MemoryEntry, } -#[derive( - Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, TS, Default, -)] -pub enum ArtifactReadFamily { - #[serde(rename = "memory")] - #[default] - Memory, - #[serde(rename = "atomic-memory")] - AtomicMemory, - #[serde(rename = "experience")] - Experience, - #[serde(rename = "skill")] - Skill, - #[serde(rename = "handoff")] - Handoff, - #[serde(rename = "profile")] - Profile, - #[serde(rename = "prompt")] - Prompt, - #[serde(rename = "topic-memory")] - TopicMemory, -} - -impl std::fmt::Display for ArtifactReadFamily { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Memory => write!(f, "memory"), - Self::AtomicMemory => write!(f, "atomic-memory"), - Self::Experience => write!(f, "experience"), - Self::Skill => write!(f, "skill"), - Self::Handoff => write!(f, "handoff"), - Self::Profile => write!(f, "profile"), - Self::Prompt => write!(f, "prompt"), - Self::TopicMemory => write!(f, "topic-memory"), - } - } -} - #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, TS)] #[serde(deny_unknown_fields)] pub struct ArtifactReference { @@ -435,9 +397,19 @@ pub struct ArtifactRevision { #[serde(rename = "scope_id")] pub scope_id: String, #[serde(rename = "family")] - pub family: ArtifactReadFamily, + pub family: String, #[serde(rename = "artifact_id")] pub artifact_id: String, + /// Persisted schema version for a custom Artifact family. + #[ts(optional)] + #[ts(type = "string | null")] + #[serde( + rename = "schema_version", + default, + with = "::serde_with::rust::double_option", + skip_serializing_if = "Option::is_none" + )] + pub schema_version: Option>, #[serde(rename = "revision")] pub revision: i64, #[ts(type = "Record")] @@ -495,7 +467,7 @@ impl ArtifactRevision { #[allow(clippy::too_many_arguments)] pub fn new( scope_id: String, - family: ArtifactReadFamily, + family: String, artifact_id: String, revision: i64, content: std::collections::HashMap, @@ -507,6 +479,7 @@ impl ArtifactRevision { scope_id, family, artifact_id, + schema_version: None, revision, content, sources, @@ -648,6 +621,12 @@ impl std::fmt::Display for AtomicMemoryState { #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, TS)] #[serde(deny_unknown_fields)] pub struct Capabilities { + #[ts(optional)] + #[serde( + rename = "custom_artifact_schemas", + skip_serializing_if = "Option::is_none" + )] + pub custom_artifact_schemas: Option>, #[ts(optional)] #[serde(rename = "prompts", skip_serializing_if = "Option::is_none")] pub prompts: Option>, @@ -713,6 +692,7 @@ impl Capabilities { context_versions: Vec, ) -> Capabilities { Capabilities { + custom_artifact_schemas: None, prompts: None, artifact_dreaming: None, artifact_dreaming_operations: None, @@ -747,6 +727,44 @@ impl ContractVersion { } } +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, TS)] +#[serde(deny_unknown_fields)] +pub struct CustomArtifactFamilySchema { + #[serde(rename = "family")] + pub family: String, + #[serde(rename = "schema_version")] + pub schema_version: String, + #[ts(type = "Record")] + #[serde(rename = "content_schema")] + pub content_schema: std::collections::HashMap, + /// Legacy persisted text fields; new plugins use search. + #[serde(rename = "search_fields")] + pub search_fields: Vec, + /// Persistent declarative indexes and supported search modes. + #[ts(type = "Record")] + #[ts(optional)] + #[serde(rename = "search", skip_serializing_if = "Option::is_none")] + pub search: Option>, +} + +impl CustomArtifactFamilySchema { + #[allow(clippy::too_many_arguments)] + pub fn new( + family: String, + schema_version: String, + content_schema: std::collections::HashMap, + search_fields: Vec, + ) -> CustomArtifactFamilySchema { + CustomArtifactFamilySchema { + family, + schema_version, + content_schema, + search_fields, + search: None, + } + } +} + #[derive( Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, TS, Default, )] @@ -1551,7 +1569,7 @@ pub fn declarations(config: &ts_rs::Config) -> Vec { ::decl(config), ::decl(config), ::decl(config), - ::decl(config), + format!("type ArtifactReadFamily = {};", ::name(config)), ::decl(config), ::decl(config), ::decl(config), @@ -1562,6 +1580,7 @@ pub fn declarations(config: &ts_rs::Config) -> Vec { ::decl(config), ::decl(config), ::decl(config), + ::decl(config), ::decl(config), ::decl(config), ::decl(config), diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index 48789d6742..3d65a1f2fb 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -261,6 +261,131 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/admin/artifact-plugins": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Artifact plugin registrations */ + get: operations["list_artifact_plugins"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get an Artifact plugin registration */ + get: operations["get_artifact_plugin"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/enable-all": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Enable the currently disabled Artifact plugins + * @description Captures registrations and generations at request start, then accepts each eligible enable independently. Enabled, unloaded, failed and transitioning plugins are skipped. Concurrent changes and validation failures are reported per plugin without rolling back successes. Does not discover or reload plugins; poll each status URL for the asynchronous outcome. + */ + post: operations["enable_all_artifact_plugins"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/reload": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Load or update a Worker-loaded Artifact plugin + * @description Reads the binding's direct child entry under POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR and fixes its resolved immutable directory. The manifest binding_name must match the directory entry and request path. Saves validated configuration and declarations without creating a pipeline. Existing enabled or disabled plugins keep their state; initial, unloaded or failed registrations become disabled. This initial path supports the global SQLite Supervisor. Plugin code is loaded only in child processes. + */ + post: operations["reload_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/enable": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Enable an Artifact plugin */ + post: operations["enable_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/disable": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Disable an Artifact plugin */ + post: operations["disable_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/admin/artifact-plugins/{binding_name}/unload": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Unload an Artifact plugin */ + post: operations["unload_artifact_plugin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/capabilities": { parameters: { query?: never; @@ -2885,6 +3010,8 @@ export interface components { scope_id: string; family: components["schemas"]["ArtifactReadFamily"]; artifact_id: string; + /** @description Persisted schema version for a custom Artifact family. */ + schema_version?: string | null; revision: number; sources: components["schemas"]["SourceTypeReference"][]; artifacts: components["schemas"]["ArtifactReference"][]; @@ -2926,6 +3053,8 @@ export interface components { scope_id: string; family: components["schemas"]["ArtifactReadFamily"]; artifact_id: string; + /** @description Persisted schema version for a custom Artifact family. */ + schema_version?: string | null; revision: number; content: { [key: string]: unknown; @@ -3259,7 +3388,58 @@ export interface components { /** @enum {string} */ effect: "review_then_publish" | "review_then_publish_configuration" | "review_then_replace_tags"; }; + CustomArtifactFamilySchema: { + family: string; + schema_version: string; + content_schema: { + [key: string]: unknown; + }; + /** @description Legacy persisted text fields; new plugins use search. */ + search_fields: string[]; + /** @description Persistent declarative indexes and supported search modes. */ + search?: { + [key: string]: unknown; + }; + }; + ReloadArtifactPluginRequest: { + expected_generation: number; + }; + ChangeArtifactPluginStateRequest: { + expected_generation: number; + }; + ArtifactPluginState: { + binding_name: string; + plugin_id: string; + family: string; + binding_generation: number; + /** @enum {string} */ + desired_state: "enabled" | "disabled" | "unloaded"; + /** @enum {string} */ + phase: "activating" | "enabled" | "stopping" | "disabled" | "unloading" | "unloaded" | "failed"; + last_error: string | null; + }; + ArtifactPluginPage: { + items: components["schemas"]["ArtifactPluginState"][]; + }; + EnableArtifactPluginResult: { + binding_name: string; + /** @enum {string} */ + outcome: "accepted" | "skipped" | "failed"; + /** @description Registration status URL for the asynchronous result. */ + status_url: string; + state: components["schemas"]["ArtifactPluginState"]; + /** + * @description Safe per-plugin failure code; null for accepted or skipped entries. + * @enum {string|null} + */ + error: "conflict" | "invalid_plugin" | "unavailable" | null; + }; + EnableAllArtifactPluginsResponse: { + items: components["schemas"]["EnableArtifactPluginResult"][]; + }; Capabilities: { + /** @default [] */ + custom_artifact_schemas: components["schemas"]["CustomArtifactFamilySchema"][]; /** @default {} */ prompts: { [key: string]: components["schemas"]["PromptCapability"]; @@ -4784,8 +4964,32 @@ export interface components { [key: string]: unknown; }; }; + /** @description Existing embedding space identity shared with Memory; no additional modality or version fields. */ + EmbeddingProfile: { + profile_id: string; + model: string; + dimension: number; + /** + * @default l2 + * @enum {string} + */ + distance: "l2"; + /** + * @default unit + * @enum {string} + */ + normalization: "none" | "unit"; + }; SearchArtifactsRequest: { - query: string; + query?: string; + query_vector?: { + profile: components["schemas"]["EmbeddingProfile"]; + vector: number[]; + }; + indexes?: { + text?: string[]; + vector?: string[]; + }; /** @default 10 */ limit: number; mode?: string; @@ -5182,8 +5386,7 @@ export interface components { CaptureStatus: "accepted"; /** @enum {string} */ BaseArtifactFamily: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"; - /** @enum {string} */ - ArtifactReadFamily: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"; + ArtifactReadFamily: string; /** @enum {string} */ StatsPeriod: "today" | "7d" | "30d"; /** @enum {string} */ @@ -6315,6 +6518,217 @@ export interface operations { 503: components["responses"]["Unavailable"]; }; }; + list_artifact_plugins: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Durable registration states. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginPage"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 503: components["responses"]["Unavailable"]; + }; + }; + get_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Durable registration state without plugin configuration secrets. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 503: components["responses"]["Unavailable"]; + }; + }; + enable_all_artifact_plugins: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Per-plugin acceptance, skip or failure results. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EnableAllArtifactPluginsResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 503: components["responses"]["Unavailable"]; + }; + }; + reload_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReloadArtifactPluginRequest"]; + }; + }; + responses: { + /** @description Definition saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + enable_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangeArtifactPluginStateRequest"]; + }; + }; + responses: { + /** @description Target saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + disable_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangeArtifactPluginStateRequest"]; + }; + }; + responses: { + /** @description Target saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + unload_artifact_plugin: { + parameters: { + query?: never; + header?: never; + path: { + binding_name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangeArtifactPluginStateRequest"]; + }; + }; + responses: { + /** @description Target saved; poll its registration state until transition completes. */ + 202: { + headers: { + /** @description Registration status URL. */ + Location?: string; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ArtifactPluginState"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; get_capabilities: { parameters: { query?: never; diff --git a/desktop/ui/src/generated/ipc.ts b/desktop/ui/src/generated/ipc.ts index b1cfde06ed..8afeca8380 100644 --- a/desktop/ui/src/generated/ipc.ts +++ b/desktop/ui/src/generated/ipc.ts @@ -121,9 +121,13 @@ export type AccessResourceType = "server" | "scope" | "artifact"; export type AccessRole = "handoff.viewer" | "handoff.receiver" | "artifact.viewer" | "prompt.user" | "artifact.owner" | "scope.viewer" | "scope.contributor" | "scope.reviewer" | "scope.delegator" | "scope.admin" | "server.observer" | "server.admin"; export type ArtifactDreamingOperationCapability = { operation: DreamOperation, output_kind: OutputKind, effect: Effect, }; export type ArtifactFamilyAccessCapability = { family: string, enabled: boolean, share_unit: ShareUnit, actions: Array, grantable_roles: Array, }; -export type ArtifactReadFamily = "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"; +export type ArtifactReadFamily = string; export type ArtifactReference = { family: string, artifact_id: string, revision: number, }; -export type ArtifactRevision = { scope_id: string, family: ArtifactReadFamily, artifact_id: string, revision: number, content: Record, sources: Array, artifacts: Array, content_digest: string, +export type ArtifactRevision = { scope_id: string, family: string, artifact_id: string, +/** + * Persisted schema version for a custom Artifact family. + */ +schema_version?: string | null, revision: number, content: Record, sources: Array, artifacts: Array, content_digest: string, /** * When this revision was committed. Absent for revisions stored before this field existed. */ @@ -149,7 +153,7 @@ score: number, matched_by: Array, }; export type AtomicMemorySearchMode = "text" | "vector" | "hybrid"; export type AtomicMemoryState = "active" | "forgotten" | "merged" | "retired"; export type BuiltinProfile = "coding" | "conversation"; -export type Capabilities = { prompts?: { [key in string]: PromptCapability }, +export type Capabilities = { custom_artifact_schemas?: Array, prompts?: { [key in string]: PromptCapability }, /** * Whether asynchronous Artifact Dream execution is configured. */ @@ -180,6 +184,15 @@ external_skill_registry?: boolean, handoff_generation: boolean, search_modes: Array, context_versions: Array, }; export type Configuration = "configured" | "unconfigured" | "unknown"; export type ContractVersion = { major: number, minor: number, }; +export type CustomArtifactFamilySchema = { family: string, schema_version: string, content_schema: Record, +/** + * Legacy persisted text fields; new plugins use search. + */ +search_fields: Array, +/** + * Persistent declarative indexes and supported search modes. + */ +search?: Record, }; export type DreamOperation = "refine_experience" | "derive_skill" | "revise_skill" | "revise_profile" | "revise_memory" | "revise_topic_memory" | "refresh_handoff" | "revise_prompt" | "revise_tags"; export type Effect = "review_then_publish" | "review_then_publish_configuration" | "review_then_replace_tags"; export type ExtractionBackground = { diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index 8447802e08..24763e365e 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "bce4e0d3deec56590a29c3c06af4d853fc8cad2917d8832b23c92dd160472ccd"; +export const contractSha256 = "37936d7897ec2d981dcc195d3b7b7a9f99c572d866c0193b37df43617077c155"; export const operations = { "get_liveness": { "method": "GET", diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 53ffa29817..3fa36ab702 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -269,7 +269,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_source_write_plugins,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_source_write_plugins,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ -f ``` @@ -277,7 +277,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions' \ + --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions,pc_artifact_vector_indexes,pc_artifact_index_changes' \ -f ``` @@ -289,17 +289,21 @@ so the previous database remains available for recovery: -f ``` - After Layer 3 completes successfully, import Profile policy in Layer 4: + After Layer 3 completes successfully, import Profile policy and Artifact search projections in Layer 4: ```bash obloader -D --csv \ - --table 'pc_profile_policies' \ + --table 'pc_profile_policies,pc_artifact_index_coverage,pc_artifact_search_text' \ -f ``` Wait for each invocation to complete successfully before starting the next. Treat any OBLoader error, bad record, or conflict record as a failed restore. Order within a layer is irrelevant because no table in a layer references another table in the same layer. + `pc_artifact_vectors` is SQLite-only. For OceanBase/seekDB, restore registered `pc_av_` row data after Layer 4: + use database initialization to create and validate the restored registered spaces first. Preserve the exact + `space_id` and `build_id` identities; never regenerate them from restored content before importing vector rows. + Keep all writers stopped until the fixed schema, native spaces and restored index state are verified. 6. If the installation has additional PowerContext-managed tables not listed above, these tested layers do not classify them. Inspect their foreign-key constraints and place each table after all of its parents; do not add them to an all-table invocation. diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 21ce571dd0..18d9617449 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -260,7 +260,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_source_write_plugins,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_source_write_plugins,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions,pc_artifact_family_schemas,pc_artifact_vector_spaces,pc_artifact_search_states' \ -f ``` @@ -268,7 +268,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions' \ + --table 'pc_dream_runs,pc_context_bootstrap_receipts,pc_scope_context_references,pc_scope_external_references,pc_scope_creation_requests,pc_scope_settings,pc_scope_bindings,pc_artifact_handles,pc_artifact_heads,pc_artifact_lineage_sources,pc_artifact_lineage_artifacts,pc_artifact_publications,pc_candidate_versions,pc_topic_memory_revision_publications,pc_memory_entry_versions,pc_artifact_vector_indexes,pc_artifact_index_changes' \ -f ``` @@ -280,16 +280,20 @@ collation，但不会包含数据库 URL 或凭据。 -f ``` - 第 3 层成功完成后，导入第 4 层中的 Profile Policy： + 第 3 层成功完成后，导入第 4 层中的 Profile Policy 和 Artifact 检索投影： ```bash obloader -D --csv \ - --table 'pc_profile_policies' \ + --table 'pc_profile_policies,pc_artifact_index_coverage,pc_artifact_search_text' \ -f ``` 每个命令成功完成后才能开始下一层。OBLoader 出现任何错误、bad record 或 conflict record 时，都应判定恢复 失败。同一层内的表互不引用，因此层内顺序无关。 + `pc_artifact_vectors` 仅用于 SQLite。OceanBase/seekDB 在第 4 层之后恢复已登记的 `pc_av_` 行数据： + 先通过数据库初始化创建并校验已恢复登记的空间物理表。 + 保留精确的 `space_id` 和 `build_id`；导入向量行之前，不从内容重新分配这些身份。 + 固定 schema、原生空间和已恢复索引状态全部验证完成前，保持所有写入方停止。 6. 如果安装中还存在上面未列出的 PowerContext 管理表，则这些已测试的层并未对它们分类。检查其外键约束，将每张 表放在其所有父表之后；不要把它们加入全表导入命令。 7. 逐表比较源数据库和目标数据库的记录数，检查 identity column collation，并测试仅大小写或重音不同的 diff --git a/e2e/bub/tests/test_access_audit.py b/e2e/bub/tests/test_access_audit.py index 5df0ec58d8..0cf4e5e778 100644 --- a/e2e/bub/tests/test_access_audit.py +++ b/e2e/bub/tests/test_access_audit.py @@ -260,6 +260,13 @@ def test_a_handoff_report_over_every_scope_reaches_them_although_it_is_authorize "set_scope_binding", "clear_scope_binding", "register_source_definition", + "list_artifact_plugins", + "get_artifact_plugin", + "enable_all_artifact_plugins", + "reload_artifact_plugin", + "enable_artifact_plugin", + "disable_artifact_plugin", + "unload_artifact_plugin", "load_source_write_plugin", "source_configure", "pc_source_instance_set_enabled", diff --git a/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md b/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md index 6c717801bd..d07555502e 100644 --- a/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md +++ b/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md @@ -87,12 +87,12 @@ not replace those schemas or enforce their constraints. It contains no scenario ## mcp__powercontext__get_artifact -- Required: `artifact_id: string`, `family: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"`, `scope_id: string`. +- Required: `artifact_id: string`, `family: ArtifactReadFamily`, `scope_id: string`. - Optional: `If-None-Match: string`. ## mcp__powercontext__get_artifact_revision -- Required: `artifact_id: string`, `family: "memory" | "atomic-memory" | "experience" | "skill" | "handoff" | "profile" | "prompt" | "topic-memory"`, `revision: integer`, `scope_id: string`. +- Required: `artifact_id: string`, `family: ArtifactReadFamily`, `revision: integer`, `scope_id: string`. - Optional: (none). ## mcp__powercontext__get_atomic_memory_state diff --git a/examples/artifact_plugins/README.md b/examples/artifact_plugins/README.md new file mode 100644 index 0000000000..70526c943d --- /dev/null +++ b/examples/artifact_plugins/README.md @@ -0,0 +1,221 @@ +# 用户自定义 Artifact 插件示例 + +用户把插件部署到统一发现目录。PowerContext Server 启动时校验新插件并保存定义，首次加载为 `disabled`；管理员启用后，Supervisor 按清单参数派发 Worker。Worker 加载 pipeline，框架保存生成结果，应用通过公开 API 读取。 + +| 模板目录 | family / binding | 生成结果 | 业务配置 | +| --- | --- | --- | --- | +| `incident_report` | `incident-report` / `incident-reports` | 每条 Source 一份工单报告 | `title_prefix` | +| `ticket_summary` | `ticket-summary` / `ticket-summaries` | 每个 Source 窗口一份文本摘要 | `language` | + +本示例使用文件型 SQLite、global Supervisor、Server 的 all 角色、Worker 内加载 Python 和声明式文本检索，不依赖 LLM。内存型 SQLite 可以加载声明，但启用生成会返回 `422`，因为 Worker 子进程无法共享 Server 的内存数据库。 + +## 1. 配置并启动 Server + +在仓库根目录执行： + +```bash +uv sync --extra server --extra cli +export POWERCONTEXT_HOME="$PWD/.powercontext/artifact-plugin-example" +export POWERCONTEXT_SERVER_DATABASE='{"kind":"sqlite","url":"sqlite+aiosqlite:///.powercontext/artifact-plugin-example.db"}' +export POWERCONTEXT_SERVER_RUNTIME='{"artifact_processing_families":[]}' +export POWERCONTEXT_SERVER_AUTH_ENABLED=true +export POWERCONTEXT_SERVER_AUTH_TOKEN=local-example-admin +export POWERCONTEXT_SERVER_ACCESS_MODE=enforced +export POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR="$POWERCONTEXT_HOME/plugins" +mkdir -p "$POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR" +uv run powercontext server run --no-env-file --host 127.0.0.1 --port 8000 +``` + +Server 的 `.env` / envfile 保存数据库、监听、鉴权、共用模型和统一插件目录等全局设置。`POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR` 沿用现有加载规则：环境变量优先于 envfile。各插件的 Scope、批次、调度和业务参数只保存在各自的 `plugin.yml`。 + +以下命令在仓库根目录的另一个终端运行，需要 `curl` 和 `jq`。首次使用先创建 Scope；已有 Scope 的部署可以在启动前准备好插件目录。 + +```bash +export PC_URL=http://127.0.0.1:8000 +export PC_AUTH='Authorization: Bearer local-example-admin' +export PC_PLUGIN_ROOT="$PWD/.powercontext/artifact-plugin-example/plugins" +export PC_VERSION_ROOT="$PWD/.powercontext/artifact-plugin-example/versions" +export PC_REPORT_SCOPE=$(curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/scopes" -d '{"title":"Reports","summary":"Incident reports","idempotency_key":"artifact-reports-example"}' \ + | jq -r .scope_id) +export PC_SUMMARY_SCOPE=$(curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/scopes" -d '{"title":"Summaries","summary":"Ticket summaries","idempotency_key":"artifact-summaries-example"}' \ + | jq -r .scope_id) +``` + +## 2. 部署插件清单和代码 + +每个插件目录有一份完整清单。报告插件示例如下，显式填写的 `scope_ids` 必须对应实际存在的 Scope： + +```yaml +id: example-incident-report +version: "1" +entry: ./plugin.py:create_pipeline +binding_name: incident-reports +scope_ids: [team-a] +max_sources: 20 +processing: + interval_seconds: 1 + max_workers: 1 + timeout_seconds: 60 +config: + title_prefix: 工单快照报告 +``` + +`entry` 相对清单所在目录解析。统一目录只发现直接子目录，目录项名称必须等于 `binding_name`。版本目录放在发现目录之外，通过符号链接提供当前入口；Server 保存解析后的实际目录，切换链接不会改变已有 Worker 使用的代码。 + +Scope、窗口上限、发现间隔、并行 Worker 数和 Worker 超时只作用于对应 binding。数值必须为正数，批次和 Worker 数必须为整数。`config` 按插件的 `Config` 模型校验后交给 pipeline。清单顶层禁止 `enable` / `enabled` 字段，无论值为 true 还是 false；运行状态由管理 API 控制。含这些字段的清单会被拒绝，reload 返回 `422` 并保留已有有效定义和状态。 + +`scope_ids` 可省略：框架动态发现已有输入积压和后续新增的 Scope，并按后台身份校验访问权限。显式填写非空名单时，该 binding 只处理名单内的已有 Scope。处理范围不授予权限。空数组 `scope_ids: []` 会被拒绝。 + +摘要插件使用自己的 Scope、`max_sources=4`、5 秒间隔、2 个 Worker、90 秒超时和 `language` 配置。插件通过安装包中的 `powercontext.artifact_plugins` 声明 schema、输入契约和可选 FTS 能力。 + +```bash +mkdir -p "$PC_PLUGIN_ROOT" "$PC_VERSION_ROOT" +cp -R examples/artifact_plugins/incident_report "$PC_VERSION_ROOT/incident-report-v1" +cp -R examples/artifact_plugins/ticket_summary "$PC_VERSION_ROOT/ticket-summary-v1" +uv run python - <<'PY' +import os +from pathlib import Path +import yaml + +root = Path(os.environ["PC_VERSION_ROOT"]) +for name, scope in ( + ("incident-report-v1", os.environ["PC_REPORT_SCOPE"]), + ("ticket-summary-v1", os.environ["PC_SUMMARY_SCOPE"]), +): + path = root / name / "plugin.yml" + manifest = yaml.safe_load(path.read_text(encoding="utf-8")) + manifest["scope_ids"] = [scope] + path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8") +PY +ln -s "$PC_VERSION_ROOT/incident-report-v1" "$PC_PLUGIN_ROOT/incident-reports" +ln -s "$PC_VERSION_ROOT/ticket-summary-v1" "$PC_PLUGIN_ROOT/ticket-summaries" +``` + +Artifact 插件通过输入映射选择每种 Source 的 Projection。两个示例均消费主线已有的 `content` Source： + +```python +inputs = InputContract(projections={"content": TEXT_EVIDENCE_PROJECTION_KEY}) +``` + +映射必须非空，每种类型选择一个合法的 ProjectionKey。`GenerationSource` 包含 `ref`、`projection` 和完整 JSON `value`；对象、数组、嵌套字段、数字、布尔值和 null 保持原有结构。示例调用 `TextEvidence.model_validate(source.value).content` 读取文本。 + +一个 family 可消费多个 Source 类型，一个类型也可供多个 family 消费。关联由上述输入声明和输出 family 决定，与两端插件 ID 或 binding 名称无关。`max_sources` 限制 journal 窗口条目数；未声明类型被过滤，batch 不保证所有类型齐全，不等待凑齐或自动配对。draft 只列出实际使用的精确 SourceRef。 + +对于外部 Source，Worker 读取已保存的 Projection，不加载 Source 插件代码，不请求外部最新数据。已接受类型缺少指定名称或版本时整个窗口失败，游标不推进，也不使用其他投影代替。输入预算按完整批次 JSON 证据计量，当前上限为 1 MiB；超限不截断或部分提交，可通过新目录和 reload 调整窗口大小。 + +## 3. 加载并启用 + +启动扫描会加载首次发现的插件。Server 已运行时，新增插件需要显式 reload；请求只携带 `expected_generation`，首次为 0： + +```bash +for binding in incident-reports ticket-summaries; do + curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/admin/artifact-plugins/$binding/reload" -d '{"expected_generation":0}' +done +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/admin/artifact-plugins" | jq +curl -fsS -X POST -H "$PC_AUTH" "$PC_URL/v1/admin/artifact-plugins/enable-all" | jq +``` + +reload 返回 `202` 和 `Location`。首次加载只校验入口、Config、输入输出声明、Scope 与身份，保存完整定义和读取能力；不调用生成工厂或生成方法。查询状态到 `disabled` 后即可启用。 + +enable-all 无需请求体，只处理请求开始时已加载且 disabled 的插件，不扫描目录。逐插件返回 `accepted`、`skipped` 或 `failed`、安全错误码和 `status_url`。一个失败不回滚其他成功项；已启用、卸载、failed 或转换中的插件跳过。通过状态 URL 查询到 `phase=enabled` 才表示异步装配完成。首次加载失败不产生注册记录，启动日志报告该目录的失败，其他合法插件继续加载。 + +## 4. 写入 Source 并读取结果 + +```bash +for scope in "$PC_REPORT_SCOPE" "$PC_SUMMARY_SCOPE"; do + jq -n --arg scope "$scope" \ + '{scope_id:$scope,source_id:"ticket-1",content:"Outage: database recovered after retry."}' \ + | curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/sources/content" --data-binary @- +done +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report" | jq +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/scopes/$PC_SUMMARY_SCOPE/artifacts/ticket-summary" | jq +curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report/search" -d '{"query":"Outage"}' | jq +export PC_ARTIFACT=$(curl -fsS -H "$PC_AUTH" \ + "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report" | jq -r '.items[0].artifact_id') +curl -fsS -H "$PC_AUTH" "$PC_URL/v1/scopes/$PC_REPORT_SCOPE/artifacts/incident-report/$PC_ARTIFACT" | jq +``` + +生成异步执行，尚无产物时 list / search 可返回空结果；已有产物但首次索引尚未就绪时，search 返回 `503`，稍后重试。list 返回 Artifact 身份，get 返回内容、schema 版本和精确 lineage。启用时从持久游标消费历史与停用期间积压；重复写入相同 Source 不产生重复修订，生成失败不推进游标或提交部分结果。 + +两个示例通过 `ArtifactFamily.search` 和 `TextIndex` 声明检索字段。文本检索支持中文、英文和混合文本，只匹配当前 revision 对应 schema 声明的字段。统一 `/search` 返回 `results`，包含完整 `content`、精确 revision 和 lineage；`include_scores` 可附带评分。 + +Server 根据持久声明恢复索引，不要求生成插件启用或代码目录存在。首次构建尚未就绪时返回 `503`；已发布索引随写入同步维护。检索表由统一数据库初始化创建，无需单独安装。 + +向量、多片段及 hybrid 通过 `ArtifactSearchDeclaration`、`VectorIndex`（`source="text"` 或 `source="content"`） 声明。OceanBase/seekDB 热加载新向量 profile 后，需要重新初始化数据库（例如重启 Server）以创建对应空间。 + +## 5. 启停、更新、卸载与恢复 + +单插件管理使用最新 GET 返回的 `binding_generation`，过期代次返回 `409`： + +```bash +export PC_GENERATION=$(curl -fsS -H "$PC_AUTH" \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports" | jq -r .binding_generation) +curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports/disable" \ + -d "{\"expected_generation\":$PC_GENERATION}" +``` + +查询到 `disabled` 后再读取新代次，可调用 `enable` 或 `unload`。启停不修改清单。卸载回收 Worker 并移除执行定义，保留身份、代次、schema、读取能力、消费进度和已有结果；确认 `unloaded` 后可删除专属版本目录，再次加载需 reload。 + +更新代码或配置时准备完整的新版本目录，再切换发现入口并调用 reload： + +```bash +cp -R "$PC_VERSION_ROOT/incident-report-v1" "$PC_VERSION_ROOT/incident-report-v2" +uv run python - <<'PY' +import os +from pathlib import Path +import yaml + +path = Path(os.environ["PC_VERSION_ROOT"]) / "incident-report-v2/plugin.yml" +manifest = yaml.safe_load(path.read_text(encoding="utf-8")) +manifest["version"] = "2" +manifest["config"]["title_prefix"] = "新版工单报告" +path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8") +PY +ln -s "$PC_VERSION_ROOT/incident-report-v2" "$PC_VERSION_ROOT/incident-reports-next" +mv -Tf "$PC_VERSION_ROOT/incident-reports-next" "$PC_PLUGIN_ROOT/incident-reports" +export PC_GENERATION=$(curl -fsS -H "$PC_AUTH" \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports" | jq -r .binding_generation) +curl -fsS -H "$PC_AUTH" -H 'Content-Type: application/json' \ + "$PC_URL/v1/admin/artifact-plugins/incident-reports/reload" \ + -d "{\"expected_generation\":$PC_GENERATION}" +``` + +候选校验失败保留原定义和状态。接受更新后撤销旧代次提交资格，回收旧 Worker，再装配新定义；确认完成前保留旧目录。reload 更新正常插件保留原启停状态；卸载或 failed 后加载为 disabled，需要再次启用。同一 `(family, schema_version)` 的内容 schema 不可变。已有检索 family 的 schema 身份、索引定义和版本冻结；无检索 family 可以演进内容 schema，但不能补加检索。允许调整 `modes`、`default_mode`、`defaults`、`candidate_limit`、`rrf_k`、`text_weight`、`vector_weight` 查询策略，更新不会触发索引重建、停搜或重新编码。 + +Server 重启恢复持久快照与已保存的启停、卸载状态，并继续未完成转换。扫描只发现新插件，不覆盖已有配置或重新加载卸载记录；failed 等待管理员 reload 修复。恢复 enabled 会更新代次，管理操作始终先 GET。无需每次重启调用 enable-all，结果读取不依赖插件进程。 + +## 6. 通用检索与精确证据读取 + +上层应用 / Agent 通过公开的 Artifact get / list / 精确 revision 和 FTS 接口获取数据，自行理解问题、组合证据并计算距离、次数或排名。PowerContext 提供存储、生成、版本、鉴权、检索和精确证据读取，不执行插件业务查询回调。 + +生成插件通过 `GenerationContext` 使用同一套读取与检索语义： + +| 接口 | 语义 | +| --- | --- | +| `read_latest(family, artifact_id)` | 读取当前 revision，缺失时返回 `None`；独立调用不形成跨 family 快照 | +| `read_revision(ref)` | 读取精确 revision，支持已被新 head 替代的历史内容；缺失时报错 | +| `list_artifacts(family, limit=50, cursor=None)` | 分页返回可见 active head 的引用；每页上限 100，游标只在本次生成上下文内使用 | +| `search_artifacts(family, query=None, **parameters)` | 经共享 ArtifactSearch 返回完整 Artifact，支持已声明模式和索引；limit 默认 10，上限 200 | + +生成默认可读自己的输出 family；需要跨 family 时，使用 `InputContract(..., artifact_families=(...))` 声明额外范围。所有读取固定在当前 Scope，并受 Worker 身份权限约束。get / list 支持已注册 Artifact family；内置 family 的快照没有自定义 `schema_version` 时返回 `None`。生成 Worker 与 HTTP / SDK 遵守相同的检索语义；无检索能力的已知 family 明确返回不支持。 + +list / search 不自动登记为生成证据。search 返回完整正文；需要用于 draft 的 lineage 或更新基准时，插件仍须显式调用 read_revision 读取所选精确版本，登记实际使用的证据。完整枚举应遍历分页 list 或持久清单，不能用 top-k 表示完整集合。历史读取先固定发布根，再沿清单读取精确引用；当前 FTS 不包含旧 revision，不能据此排除历史对象。 + +统一 HTTP search 复用共享检索服务。插件停用或卸载后，已保存内容的 get / list / 声明式检索继续可用。普通读取与检索不推进 Source 消费进度。 + +## 端到端验收 + +```bash +uv run pytest tests/e2e/test_artifact_plugins.py \ + tests/e2e/test_artifact_plugin_search.py -q +``` + +测试启动安装包提供的 `powercontext server run`，通过真实 envfile / 环境变量、部署目录和 HTTP 完成加载、启用、消费与查询。覆盖首次加载不生成、多插件独立配置、批量部分失败和并发冲突、reload、单插件启停 / 卸载、重启与失败恢复。 + +测试只使用产品的公开路径。 diff --git a/examples/artifact_plugins/incident_report/plugin.py b/examples/artifact_plugins/incident_report/plugin.py new file mode 100644 index 0000000000..d612266039 --- /dev/null +++ b/examples/artifact_plugins/incident_report/plugin.py @@ -0,0 +1,97 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate one deterministic report for each exact ticket snapshot.""" + +import hashlib +import json + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.artifact_plugins import ( + ArtifactDraft, + ArtifactFamily, + ArtifactSearchDeclaration, + GenerationBatch, + GenerationContext, + InputContract, + TextIndex, +) +from powercontext.sources import TEXT_EVIDENCE_PROJECTION_KEY, TextEvidence + +plugin_api_version = "1" + + +class Config(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + title_prefix: str = "Incident report" + fail_on_text: str | None = None + + +class IncidentReport(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + title: str = Field(min_length=1) + evidence_text: str = Field(min_length=1) + + +families = ( + ArtifactFamily( + family="incident-report", + schema_version="1", + content_model=IncidentReport, + search=ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="lexical", projection_version="1", fields=("title", "evidence_text")),), + modes=("text",), + default_mode="text", + ), + ), +) +inputs = InputContract( + projections={ + "content": TEXT_EVIDENCE_PROJECTION_KEY, + } +) + + +class IncidentReportPipeline: + def __init__(self, config: Config) -> None: + self.config = config + + async def generate(self, batch: GenerationBatch) -> list[ArtifactDraft]: + drafts = [] + for source in batch.sources: + evidence = TextEvidence.model_validate(source.value) + if self.config.fail_on_text and self.config.fail_on_text in evidence.content: + raise ValueError("Demo generation failure: no drafts from this window will be committed") # noqa: TRY003 + identity = json.dumps(source.ref.model_dump(mode="json"), sort_keys=True, separators=(",", ":")) + drafts.append( + ArtifactDraft( + family="incident-report", + schema_version="1", + artifact_id=hashlib.sha256(identity.encode()).hexdigest(), + content=IncidentReport( + title=f"{self.config.title_prefix}: {source.ref.source_id}", + evidence_text=evidence.content, + ).model_dump(mode="json"), + source_refs=(source.ref,), + ) + ) + return drafts + + +async def create_pipeline(ctx: GenerationContext, config: Config) -> IncidentReportPipeline: + return IncidentReportPipeline(config) diff --git a/examples/artifact_plugins/incident_report/plugin.yml b/examples/artifact_plugins/incident_report/plugin.yml new file mode 100644 index 0000000000..68313a4b35 --- /dev/null +++ b/examples/artifact_plugins/incident_report/plugin.yml @@ -0,0 +1,26 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: example-incident-report +version: "1" +entry: ./plugin.py:create_pipeline +binding_name: incident-reports +scope_ids: [team-a] +max_sources: 20 +processing: + interval_seconds: 1 + max_workers: 1 + timeout_seconds: 60 +config: + title_prefix: 工单快照报告 diff --git a/examples/artifact_plugins/ticket_summary/plugin.py b/examples/artifact_plugins/ticket_summary/plugin.py new file mode 100644 index 0000000000..cbb66d54fc --- /dev/null +++ b/examples/artifact_plugins/ticket_summary/plugin.py @@ -0,0 +1,95 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Summarize each Source window without an external model dependency.""" + +import hashlib +import json + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.artifact_plugins import ( + ArtifactDraft, + ArtifactFamily, + ArtifactSearchDeclaration, + GenerationBatch, + GenerationContext, + InputContract, + TextIndex, +) +from powercontext.sources import TEXT_EVIDENCE_PROJECTION_KEY, TextEvidence + +plugin_api_version = "1" + + +class Config(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + language: str = Field(default="简体中文", min_length=1) + + +class TicketSummary(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + language: str + source_ids: list[str] = Field(min_length=1) + summary_text: str = Field(min_length=1) + + +families = ( + ArtifactFamily( + family="ticket-summary", + schema_version="1", + content_model=TicketSummary, + search=ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="lexical", projection_version="1", fields=("summary_text",)),), + modes=("text",), + default_mode="text", + ), + ), +) +inputs = InputContract( + projections={ + "content": TEXT_EVIDENCE_PROJECTION_KEY, + } +) + + +class TicketSummaryPipeline: + def __init__(self, config: Config) -> None: + self.config = config + + async def generate(self, batch: GenerationBatch) -> list[ArtifactDraft]: + refs = tuple(source.ref for source in batch.sources) + identity = json.dumps([ref.model_dump(mode="json") for ref in refs], sort_keys=True, separators=(",", ":")) + return [ + ArtifactDraft( + family="ticket-summary", + schema_version="1", + artifact_id=hashlib.sha256(identity.encode()).hexdigest(), + content=TicketSummary( + language=self.config.language, + source_ids=[ref.source_id for ref in refs], + summary_text="\n".join( + TextEvidence.model_validate(source.value).content for source in batch.sources + ), + ).model_dump(mode="json"), + source_refs=refs, + ) + ] + + +async def create_pipeline(ctx: GenerationContext, config: Config) -> TicketSummaryPipeline: + return TicketSummaryPipeline(config) diff --git a/examples/artifact_plugins/ticket_summary/plugin.yml b/examples/artifact_plugins/ticket_summary/plugin.yml new file mode 100644 index 0000000000..3bf122fc71 --- /dev/null +++ b/examples/artifact_plugins/ticket_summary/plugin.yml @@ -0,0 +1,26 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: example-ticket-summary +version: "1" +entry: ./plugin.py:create_pipeline +binding_name: ticket-summaries +scope_ids: [team-b] +max_sources: 4 +processing: + interval_seconds: 5 + max_workers: 2 + timeout_seconds: 90 +config: + language: 简体中文 diff --git a/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs b/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs index b663b52c71..6254238068 100644 --- a/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs +++ b/integrations/cindy/plugins/powercontext-hooks/node/worker.cjs @@ -257,6 +257,83 @@ const OPERATIONS = { successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { + method: "GET", + path: "/v1/admin/artifact-plugins", + location: null, + scopeMode: "none", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + get_artifact_plugin: { + method: "GET", + path: "/v1/admin/artifact-plugins/{binding_name}", + location: null, + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + enable_all_artifact_plugins: { + method: "POST", + path: "/v1/admin/artifact-plugins/enable-all", + location: null, + scopeMode: "none", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + reload_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/reload", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + enable_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/enable", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + disable_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/disable", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + unload_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/unload", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, get_capabilities: { method: "GET", path: "/v1/capabilities", diff --git a/integrations/dify/plugin/powercontext_dify/contract.json b/integrations/dify/plugin/powercontext_dify/contract.json index d247ce4f42..c890d55032 100644 --- a/integrations/dify/plugin/powercontext_dify/contract.json +++ b/integrations/dify/plugin/powercontext_dify/contract.json @@ -912,16 +912,9 @@ }, "ArtifactReadFamily": { "type": "string", - "enum": [ - "memory", - "atomic-memory", - "experience", - "skill", - "handoff", - "profile", - "prompt", - "topic-memory" - ] + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z][a-z0-9-]*$" }, "ArtifactReference": { "type": "object", @@ -978,6 +971,17 @@ "maxLength": 128, "pattern": "^[\\x21-\\x7E]+$" }, + "schema_version": { + "anyOf": [ + { + "type": "string", + "description": "Persisted schema version for a custom Artifact family." + }, + { + "type": "null" + } + ] + }, "revision": { "type": "integer", "minimum": 1 diff --git a/integrations/dify/plugin/tools/pc_memory_get.yaml b/integrations/dify/plugin/tools/pc_memory_get.yaml index c02c011377..ffb439930a 100644 --- a/integrations/dify/plugin/tools/pc_memory_get.yaml +++ b/integrations/dify/plugin/tools/pc_memory_get.yaml @@ -77,18 +77,13 @@ output_schema: family: $ref: '#/definitions/Result/properties/family' type: string - enum: &id001 - - memory - - atomic-memory - - experience - - skill - - handoff - - profile - - prompt - - topic-memory artifact_id: $ref: '#/definitions/Result/properties/artifact_id' type: string + schema_version: + $ref: '#/definitions/Result/properties/schema_version' + type: string + description: Persisted schema version for a custom Artifact family. revision: $ref: '#/definitions/Result/properties/revision' type: number @@ -180,12 +175,19 @@ output_schema: pattern: .*\S.* family: type: string - enum: *id001 + minLength: 1 + maxLength: 128 + pattern: ^[a-z][a-z0-9-]*$ artifact_id: type: string minLength: 1 maxLength: 128 pattern: ^[\x21-\x7E]+$ + schema_version: + anyOf: + - type: string + description: Persisted schema version for a custom Artifact family. + - type: 'null' revision: type: integer minimum: 1 diff --git a/integrations/dify/plugin/tools/pc_memory_revise.yaml b/integrations/dify/plugin/tools/pc_memory_revise.yaml index 2353764748..178b0407f3 100644 --- a/integrations/dify/plugin/tools/pc_memory_revise.yaml +++ b/integrations/dify/plugin/tools/pc_memory_revise.yaml @@ -137,18 +137,13 @@ output_schema: family: $ref: '#/definitions/Result/properties/family' type: string - enum: &id001 - - memory - - atomic-memory - - experience - - skill - - handoff - - profile - - prompt - - topic-memory artifact_id: $ref: '#/definitions/Result/properties/artifact_id' type: string + schema_version: + $ref: '#/definitions/Result/properties/schema_version' + type: string + description: Persisted schema version for a custom Artifact family. revision: $ref: '#/definitions/Result/properties/revision' type: number @@ -229,12 +224,19 @@ output_schema: pattern: .*\S.* family: type: string - enum: *id001 + minLength: 1 + maxLength: 128 + pattern: ^[a-z][a-z0-9-]*$ artifact_id: type: string minLength: 1 maxLength: 128 pattern: ^[\x21-\x7E]+$ + schema_version: + anyOf: + - type: string + description: Persisted schema version for a custom Artifact family. + - type: 'null' revision: type: integer minimum: 1 diff --git a/integrations/dsh/plugins/powercontext/lib/index.js b/integrations/dsh/plugins/powercontext/lib/index.js index 355098a0b1..637a50b3c7 100644 --- a/integrations/dsh/plugins/powercontext/lib/index.js +++ b/integrations/dsh/plugins/powercontext/lib/index.js @@ -326,6 +326,83 @@ const OPERATIONS = { successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { + method: "GET", + path: "/v1/admin/artifact-plugins", + location: null, + scopeMode: "none", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + get_artifact_plugin: { + method: "GET", + path: "/v1/admin/artifact-plugins/{binding_name}", + location: null, + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + enable_all_artifact_plugins: { + method: "POST", + path: "/v1/admin/artifact-plugins/enable-all", + location: null, + scopeMode: "none", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, + reload_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/reload", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + enable_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/enable", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + disable_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/disable", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, + unload_artifact_plugin: { + method: "POST", + path: "/v1/admin/artifact-plugins/{binding_name}/unload", + location: "body", + scopeMode: "none", + pathParameters: ["binding_name"], + queryParams: [], + headerParams: [], + successStatuses: [202], + emptyStatuses: [] + }, get_capabilities: { method: "GET", path: "/v1/capabilities", diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index d235ab781c..6fd4138604 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -32,6 +32,13 @@ export const OPERATIONS = { get_liveness: { method: 'GET', path: '/health/live', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_readiness: { method: 'GET', path: '/health/ready', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_server_info: { method: 'GET', path: '/v1/server-info', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { method: 'GET', path: '/v1/admin/artifact-plugins', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_artifact_plugin: { method: 'GET', path: '/v1/admin/artifact-plugins/{binding_name}', location: null, scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + enable_all_artifact_plugins: { method: 'POST', path: '/v1/admin/artifact-plugins/enable-all', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + reload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/reload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + enable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/enable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + disable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/disable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + unload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/unload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, get_capabilities: { method: 'GET', path: '/v1/capabilities', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_scopes: { method: 'GET', path: '/v1/scopes', location: "query", scopeMode: 'none', pathParameters: [], queryParams: ['query','query_field','parent_scope_id','external_reference_kind','binding_integration','binding_kind','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_scope: { method: 'POST', path: '/v1/scopes', location: "body", scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index d235ab781c..6fd4138604 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -32,6 +32,13 @@ export const OPERATIONS = { get_liveness: { method: 'GET', path: '/health/live', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_readiness: { method: 'GET', path: '/health/ready', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_server_info: { method: 'GET', path: '/v1/server-info', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { method: 'GET', path: '/v1/admin/artifact-plugins', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_artifact_plugin: { method: 'GET', path: '/v1/admin/artifact-plugins/{binding_name}', location: null, scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + enable_all_artifact_plugins: { method: 'POST', path: '/v1/admin/artifact-plugins/enable-all', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + reload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/reload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + enable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/enable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + disable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/disable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + unload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/unload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, get_capabilities: { method: 'GET', path: '/v1/capabilities', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_scopes: { method: 'GET', path: '/v1/scopes', location: "query", scopeMode: 'none', pathParameters: [], queryParams: ['query','query_field','parent_scope_id','external_reference_kind','binding_integration','binding_kind','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_scope: { method: 'POST', path: '/v1/scopes', location: "body", scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index d235ab781c..6fd4138604 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -32,6 +32,13 @@ export const OPERATIONS = { get_liveness: { method: 'GET', path: '/health/live', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_readiness: { method: 'GET', path: '/health/ready', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_server_info: { method: 'GET', path: '/v1/server-info', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + list_artifact_plugins: { method: 'GET', path: '/v1/admin/artifact-plugins', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + get_artifact_plugin: { method: 'GET', path: '/v1/admin/artifact-plugins/{binding_name}', location: null, scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + enable_all_artifact_plugins: { method: 'POST', path: '/v1/admin/artifact-plugins/enable-all', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + reload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/reload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + enable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/enable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + disable_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/disable', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, + unload_artifact_plugin: { method: 'POST', path: '/v1/admin/artifact-plugins/{binding_name}/unload', location: "body", scopeMode: 'none', pathParameters: ['binding_name'], queryParams: [], headerParams: [], successStatuses: [202], emptyStatuses: [] }, get_capabilities: { method: 'GET', path: '/v1/capabilities', location: null, scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_scopes: { method: 'GET', path: '/v1/scopes', location: "query", scopeMode: 'none', pathParameters: [], queryParams: ['query','query_field','parent_scope_id','external_reference_kind','binding_integration','binding_kind','limit','cursor'], headerParams: [], successStatuses: [200], emptyStatuses: [] }, create_scope: { method: 'POST', path: '/v1/scopes', location: "body", scopeMode: 'none', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [201], emptyStatuses: [] }, diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index 3d637e34f5..3efa1913b8 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -572,6 +572,167 @@ paths: $ref: "#/components/responses/Forbidden" "503": $ref: "#/components/responses/Unavailable" + /v1/admin/artifact-plugins: + get: + tags: [artifact-plugins] + summary: List Artifact plugin registrations + operationId: list_artifact_plugins + x-powercontext-access: {action: server.admin, resource: {type: server}} + responses: + "200": + description: Durable registration states. + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginPage'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}: + get: + tags: [artifact-plugins] + summary: Get an Artifact plugin registration + operationId: get_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + responses: + "200": + description: Durable registration state without plugin configuration secrets. + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/enable-all: + post: + tags: [artifact-plugins] + summary: Enable the currently disabled Artifact plugins + description: Captures registrations and generations at request start, then accepts each eligible enable independently. Enabled, unloaded, failed and transitioning plugins are skipped. Concurrent changes and validation failures are reported per plugin without rolling back successes. Does not discover or reload plugins; poll each status URL for the asynchronous outcome. + operationId: enable_all_artifact_plugins + x-powercontext-access: {action: server.admin, resource: {type: server}} + responses: + "200": + description: Per-plugin acceptance, skip or failure results. + content: + application/json: + schema: {$ref: '#/components/schemas/EnableAllArtifactPluginsResponse'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/reload: + post: + tags: [artifact-plugins] + summary: Load or update a Worker-loaded Artifact plugin + description: Reads the binding's direct child entry under POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR and fixes its resolved immutable directory. The manifest binding_name must match the directory entry and request path. Saves validated configuration and declarations without creating a pipeline. Existing enabled or disabled plugins keep their state; initial, unloaded or failed registrations become disabled. This initial path supports the global SQLite Supervisor. Plugin code is loaded only in child processes. + operationId: reload_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ReloadArtifactPluginRequest'} + responses: + "202": + description: Definition saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/enable: + post: + tags: [artifact-plugins] + summary: Enable an Artifact plugin + operationId: enable_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ChangeArtifactPluginStateRequest'} + responses: + "202": + description: Target saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/disable: + post: + tags: [artifact-plugins] + summary: Disable an Artifact plugin + operationId: disable_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ChangeArtifactPluginStateRequest'} + responses: + "202": + description: Target saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} + /v1/admin/artifact-plugins/{binding_name}/unload: + post: + tags: [artifact-plugins] + summary: Unload an Artifact plugin + operationId: unload_artifact_plugin + x-powercontext-access: {action: server.admin, resource: {type: server}} + parameters: + - {name: binding_name, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 128, pattern: '^[a-z][a-z0-9-]*$'}} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ChangeArtifactPluginStateRequest'} + responses: + "202": + description: Target saved; poll its registration state until transition completes. + headers: + Location: {schema: {type: string}, description: Registration status URL.} + content: + application/json: + schema: {$ref: '#/components/schemas/ArtifactPluginState'} + "400": {$ref: '#/components/responses/BadRequest'} + "401": {$ref: '#/components/responses/Unauthorized'} + "403": {$ref: '#/components/responses/Forbidden'} + "404": {$ref: '#/components/responses/NotFound'} + "409": {$ref: '#/components/responses/Conflict'} + "422": {$ref: '#/components/responses/InvalidRequest'} + "503": {$ref: '#/components/responses/Unavailable'} /v1/capabilities: get: tags: [capabilities] @@ -6707,6 +6868,10 @@ components: minLength: 1 maxLength: 128 pattern: '^[\x21-\x7E]+$' + schema_version: + type: string + nullable: true + description: Persisted schema version for a custom Artifact family. revision: type: integer minimum: 1 @@ -6821,6 +6986,10 @@ components: minLength: 1 maxLength: 128 pattern: '^[\x21-\x7E]+$' + schema_version: + type: string + nullable: true + description: Persisted schema version for a custom Artifact family. revision: type: integer minimum: 1 @@ -7757,12 +7926,76 @@ components: effect: type: string enum: [review_then_publish, review_then_publish_configuration, review_then_replace_tags] + CustomArtifactFamilySchema: + type: object + additionalProperties: false + required: [family, schema_version, content_schema, search_fields] + properties: + family: {type: string} + schema_version: {type: string} + content_schema: {type: object, additionalProperties: true} + search_fields: {type: array, items: {type: string}, description: Legacy persisted text fields; new plugins use search.} + search: {type: object, additionalProperties: true, description: Persistent declarative indexes and supported search modes.} + ReloadArtifactPluginRequest: + type: object + additionalProperties: false + required: [expected_generation] + properties: + expected_generation: {type: integer, minimum: 0} + ChangeArtifactPluginStateRequest: + type: object + additionalProperties: false + required: [expected_generation] + properties: + expected_generation: {type: integer, minimum: 1} + ArtifactPluginState: + type: object + additionalProperties: false + required: [binding_name, plugin_id, family, binding_generation, desired_state, phase, last_error] + properties: + binding_name: {type: string} + plugin_id: {type: string} + family: {type: string} + binding_generation: {type: integer, minimum: 1} + desired_state: {type: string, enum: [enabled, disabled, unloaded]} + phase: {type: string, enum: [activating, enabled, stopping, disabled, unloading, unloaded, failed]} + last_error: {type: string, nullable: true} + ArtifactPluginPage: + type: object + additionalProperties: false + required: [items] + properties: + items: {type: array, items: {$ref: '#/components/schemas/ArtifactPluginState'}} + EnableArtifactPluginResult: + type: object + additionalProperties: false + required: [binding_name, outcome, status_url, state, error] + properties: + binding_name: {type: string} + outcome: {type: string, enum: [accepted, skipped, failed]} + status_url: {type: string, description: Registration status URL for the asynchronous result.} + state: {$ref: '#/components/schemas/ArtifactPluginState'} + error: + type: string + nullable: true + enum: [conflict, invalid_plugin, unavailable, null] + description: Safe per-plugin failure code; null for accepted or skipped entries. + EnableAllArtifactPluginsResponse: + type: object + additionalProperties: false + required: [items] + properties: + items: {type: array, items: {$ref: '#/components/schemas/EnableArtifactPluginResult'}} Capabilities: type: object additionalProperties: false required: [source_types, artifact_families, memory_extraction, handoff_generation, search_modes, context_versions] properties: + custom_artifact_schemas: + type: array + default: [] + items: {$ref: '#/components/schemas/CustomArtifactFamilySchema'} prompts: type: object default: {} @@ -11798,13 +12031,36 @@ components: properties: method: {type: string, minLength: 1, pattern: '.*\S.*'} params: {type: object, additionalProperties: true, default: {}} + EmbeddingProfile: + type: object + additionalProperties: false + required: [profile_id, model, dimension] + description: Existing embedding space identity shared with Memory; no additional modality or version fields. + properties: + profile_id: {type: string, minLength: 1, pattern: '.*\S.*'} + model: {type: string, minLength: 1, pattern: '.*\S.*'} + dimension: {type: integer, minimum: 1} + distance: {type: string, enum: [l2], default: l2} + normalization: {type: string, enum: [none, unit], default: unit} SearchArtifactsRequest: type: object additionalProperties: false x-powercontext-artifact-search-validation: request - required: [query] properties: query: {type: string, minLength: 1, maxLength: 8192, pattern: '.*\S.*'} + query_vector: + type: object + additionalProperties: false + required: [profile, vector] + properties: + profile: {$ref: '#/components/schemas/EmbeddingProfile'} + vector: {type: array, minItems: 1, items: {type: number}} + indexes: + type: object + additionalProperties: false + properties: + text: {type: array, minItems: 1, uniqueItems: true, items: {type: string, minLength: 1}} + vector: {type: array, minItems: 1, uniqueItems: true, items: {type: string, minLength: 1}} limit: {type: integer, minimum: 1, maximum: 200, default: 10} mode: {type: string, minLength: 1, pattern: '.*\S.*'} filters: {type: object, additionalProperties: true} @@ -12706,7 +12962,9 @@ components: enum: [memory, atomic-memory, experience, skill, handoff, profile, prompt, topic-memory] ArtifactReadFamily: type: string - enum: [memory, atomic-memory, experience, skill, handoff, profile, prompt, topic-memory] + minLength: 1 + maxLength: 128 + pattern: '^[a-z][a-z0-9-]*$' StatsPeriod: type: string enum: [today, 7d, 30d] diff --git a/release/upgrade-support.json b/release/upgrade-support.json index f29f0f4319..aea14df8d0 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.1", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "6479e5cdb3ffeba35a45b1e1d0632b7890cbc2990c593b437c2e33b66116149b", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged." + "persistence_sha256": "3c7c16fc489d6f03038cbad4dd82e14c01370761cedcb5620027551894cd7a6f", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged." }, "cases": [], "automatic_sources": { diff --git a/scripts/ci_role_smoke.py b/scripts/ci_role_smoke.py index c18980606c..9c79cd3461 100644 --- a/scripts/ci_role_smoke.py +++ b/scripts/ci_role_smoke.py @@ -18,6 +18,7 @@ import argparse import asyncio +import importlib.util import os import subprocess import sys @@ -99,8 +100,20 @@ def _check_cli(root: Path, extras: set[str]) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("extras", choices=("builtin", "cli", "builtin,cli", "server,cli")) - extras = set(parser.parse_args().extras.split(",")) + parser.add_argument("extras", choices=("core", "builtin", "cli", "builtin,cli", "server,cli")) + role = parser.parse_args().extras + extras = set() if role == "core" else set(role.split(",")) + import powercontext + + if powercontext.PowerContext is None or powercontext.ArtifactRef is None: + raise RuntimeError("Core public exports are unavailable") # noqa: TRY003 + if not extras and importlib.util.find_spec("httpx") is not None: + raise RuntimeError("Core installation unexpectedly includes httpx") # noqa: TRY003 + if extras: + from powercontext import artifact_plugins + + if powercontext.artifact_plugins is not artifact_plugins or artifact_plugins.ArtifactFamily is None: + raise RuntimeError("Plugin public export is unavailable") # noqa: TRY003 if "cli" in extras: extras.add("builtin") with tempfile.TemporaryDirectory(prefix="powercontext-role-") as temporary: diff --git a/scripts/generate_api.py b/scripts/generate_api.py index 06142b13e0..0ca464af33 100644 --- a/scripts/generate_api.py +++ b/scripts/generate_api.py @@ -127,6 +127,16 @@ def _generate_models( transport_contract.paths = {} if transport_contract.components is None or transport_contract.components.schemas is None: raise ContractGenerationError("components.schemas", None) + # Read-family values remain strings in the SDK, including custom family names. + # A referenced scalar otherwise becomes a RootModel and breaks identity comparisons. + family_schema = transport_contract.components.schemas["ArtifactReadFamily"] + if not isinstance(family_schema, Schema): + raise ContractGenerationError("ArtifactReadFamily schema", family_schema) # noqa: TRY003 + for name in ("ArtifactCollectionItem", "ArtifactRevision"): + schema = transport_contract.components.schemas[name] + if not isinstance(schema, Schema) or schema.properties is None: + raise ContractGenerationError("Artifact read schema", name) # noqa: TRY003 + schema.properties["family"] = family_schema.model_copy(deep=True) exclusive_schemas = { name: schema for name, schema in transport_contract.components.schemas.items() diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index 7448a04133..65914fe458 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -14,6 +14,10 @@ """Stable public exports for PowerContext clients.""" +from importlib import import_module +from types import ModuleType +from typing import TYPE_CHECKING + from powercontext.artifacts import ( Artifact, ArtifactAddress, @@ -113,4 +117,17 @@ "SourceStore", "Sources", "Trigger", + "artifact_plugins", ] + + +if TYPE_CHECKING: + from powercontext import artifact_plugins + + +def __getattr__(name: str) -> ModuleType: + if name == "artifact_plugins": + module = import_module("powercontext.artifact_plugins") + globals()[name] = module + return module + raise AttributeError(name) diff --git a/src/powercontext/artifact_plugins.py b/src/powercontext/artifact_plugins.py new file mode 100644 index 0000000000..ade7db50ea --- /dev/null +++ b/src/powercontext/artifact_plugins.py @@ -0,0 +1,149 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declarations and generation contracts for Worker-loaded Artifact plugins.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from contextlib import AbstractAsyncContextManager, AsyncExitStack +from dataclasses import dataclass +from typing import TypeVar + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +from powercontext.artifacts import ArtifactLineage, ArtifactRef +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, FragmentProjection, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchOutcome +from powercontext.sources import SourceProjectionKey, SourceRef + +ResourceT = TypeVar("ResourceT") + + +@dataclass(frozen=True) +class ArtifactFamily: + family: str + schema_version: str + content_model: type[BaseModel] + search: ArtifactSearchDeclaration | None = None + + +@dataclass(frozen=True) +class InputContract: + projections: dict[str, SourceProjectionKey] + artifact_families: tuple[str, ...] = () + + +class GenerationSource(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + ref: SourceRef + projection: SourceProjectionKey + value: JsonValue + + +@dataclass(frozen=True) +class GenerationBatch: + sources: tuple[GenerationSource, ...] + + +class ArtifactDraft(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + family: str + schema_version: str + artifact_id: str = Field(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$") + expected_revision: int = Field(default=0, ge=0) + content: dict[str, JsonValue] + source_refs: tuple[SourceRef, ...] + artifact_refs: tuple[ArtifactRef, ...] = () + + +class ArtifactSnapshot(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + ref: ArtifactRef + lineage: ArtifactLineage + schema_version: str | None + content: dict[str, JsonValue] + source_refs: tuple[SourceRef, ...] = () + artifact_refs: tuple[ArtifactRef, ...] = () + + +class ArtifactReferencePage(BaseModel): + items: tuple[ArtifactRef, ...] + next_cursor: str | None = None + + +class GenerationContext: + """Scope-bound reads whose exact evidence can be used by generated drafts.""" + + def __init__( + self, + scope_id: str, + resources: AsyncExitStack, + read_artifact: Callable[[str, str, int | None], Awaitable[ArtifactSnapshot | None]], + *, + list_artifacts: Callable[..., Awaitable[ArtifactReferencePage]] | None = None, + search_artifacts: Callable[..., Awaitable[ArtifactSearchOutcome]] | None = None, + ) -> None: + self.scope_id = scope_id + self._resources = resources + self._read_artifact = read_artifact + self._list_artifacts = list_artifacts + self._search_artifacts = search_artifacts + + async def enter_async_context(self, resource: AbstractAsyncContextManager[ResourceT]) -> ResourceT: + return await self._resources.enter_async_context(resource) + + async def read_latest(self, family: str, artifact_id: str) -> ArtifactSnapshot | None: + """Read one current Artifact; separate reads do not establish a cross-family snapshot.""" + + return await self._read_artifact(family, artifact_id, None) + + async def read_revision(self, ref: ArtifactRef) -> ArtifactSnapshot: + """Read and register an exact revision, including a superseded historical revision.""" + + result = await self._read_artifact(ref.family, ref.artifact_id, ref.revision) + if result is None: + raise LookupError(ref) + return result + + async def list_artifacts(self, family: str, *, limit: int = 50, cursor: str | None = None) -> ArtifactReferencePage: + """Page visible current references; use a fixed manifest for historical enumeration.""" + if self._list_artifacts is None: + raise NotImplementedError("Artifact listing is unavailable") + return await self._list_artifacts(family, limit=limit, cursor=cursor) + + async def search_artifacts(self, family: str, query: str | None = None, **parameters) -> ArtifactSearchOutcome: + """Retrieve complete exact Artifacts through the shared search contract.""" + if self._search_artifacts is None: + raise NotImplementedError("Artifact search is unavailable") + return await self._search_artifacts(family, query, **parameters) + + +__all__ = [ + "ArtifactDraft", + "ArtifactFamily", + "ArtifactReferencePage", + "ArtifactSearchDeclaration", + "ArtifactSnapshot", + "FragmentProjection", + "GenerationBatch", + "GenerationContext", + "GenerationSource", + "InputContract", + "TextIndex", + "VectorIndex", +] diff --git a/src/powercontext/artifacts/indexes.py b/src/powercontext/artifacts/indexes.py new file mode 100644 index 0000000000..91ee498e2e --- /dev/null +++ b/src/powercontext/artifacts/indexes.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declarative, persistent search projections for ordinary Artifact families.""" + +from __future__ import annotations + +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile + +Name = Annotated[str, Field(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")] +FieldName = Annotated[str, Field(min_length=1, max_length=128)] +Version = Annotated[str, Field(min_length=1, max_length=128, pattern=r".*\S.*")] +SearchMode = Literal["text", "vector", "hybrid"] + + +class _Declaration(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + +class FragmentProjection(_Declaration): + """Project an explicit persistent array; the framework does not split content.""" + + array_field: FieldName + id_field: FieldName + + +class TextIndex(_Declaration): + name: Name + kind: Literal["text"] = "text" + projection_version: Version + fields: tuple[FieldName, ...] = Field(min_length=1) + + @model_validator(mode="after") + def unique_fields(self): + if len(set(self.fields)) != len(self.fields): + raise ValueError("text fields must be distinct") # noqa: TRY003 + return self + + +class VectorIndex(_Declaration): + """Index persistent vectors or encode persistent text in one exact profile. + + For content vectors, profile_field names a persistent EmbeddingProfile object + alongside the vector. This prevents a producer replacing P1 with P2 in place. + Fragment projections resolve fields within each persistent array item. + """ + + name: Name + kind: Literal["vector"] = "vector" + projection_version: Version + source: Literal["text", "content"] + profile: EmbeddingProfile + fields: tuple[FieldName, ...] = () + vector_field: FieldName | None = None + profile_field: FieldName | None = None + fragments: FragmentProjection | None = None + + @model_validator(mode="after") + def coherent_source(self): + if not self.profile.profile_id.strip() or not self.profile.model.strip() or self.profile.dimension < 1: + raise ValueError("an explicit model, profile identity and positive dimension are required") # noqa: TRY003 + if self.source == "text": + if not self.fields or self.vector_field is not None or self.profile_field is not None: + raise ValueError("text vectors require text fields and cannot declare content vector fields") # noqa: TRY003 + if len(set(self.fields)) != len(self.fields): + raise ValueError("text fields must be distinct") # noqa: TRY003 + elif self.fields or self.vector_field is None or self.profile_field is None: + raise ValueError("content vectors require vector_field and profile_field, without text fields") # noqa: TRY003 + return self + + +SearchIndex = Annotated[TextIndex | VectorIndex, Field(discriminator="kind")] + + +class ArtifactSearchDeclaration(_Declaration): + version: Version + indexes: tuple[SearchIndex, ...] = Field(min_length=1) + modes: tuple[SearchMode, ...] = Field(min_length=1) + default_mode: SearchMode + defaults: dict[SearchMode, dict[Literal["text", "vector"], tuple[Name, ...]]] = Field(default_factory=dict) + candidate_limit: int = Field(default=200, ge=1, le=2000, strict=True) + rrf_k: int = Field(default=60, gt=0, strict=True) + text_weight: float = Field(default=1, ge=0) + vector_weight: float = Field(default=1, ge=0) + + @model_validator(mode="after") + def coherent_modes(self): + by_name = {index.name: index for index in self.indexes} + if len(by_name) != len(self.indexes) or len(set(self.modes)) != len(self.modes): + raise ValueError("index names and modes must be distinct") # noqa: TRY003 + if self.default_mode not in self.modes or set(self.defaults) - set(self.modes): + raise ValueError("defaults must refer to supported modes") # noqa: TRY003 + if self.text_weight + self.vector_weight <= 0: + raise ValueError("at least one fusion channel must have positive weight") # noqa: TRY003 + for mode in self.modes: + self.select_indexes(mode, self.defaults.get(mode)) + return self + + def select_indexes(self, mode, selection=None): # noqa: C901 - Keep the complete projection or publication invariant together. + if mode not in self.modes: + raise ValueError("search mode is not declared") # noqa: TRY003 + channels = {"text", "vector"} if mode == "hybrid" else {mode} + selection = self.defaults.get(mode) if selection is None else selection + if selection is None: + selection = {} + for channel in channels: + names = tuple(index.name for index in self.indexes if index.kind == channel) + if len(names) != 1: + raise ValueError("multiple indexes require explicit mode defaults") # noqa: TRY003 + selection[channel] = names + if set(selection) != channels: + raise ValueError("index selection must match the requested channels") # noqa: TRY003 + by_name = {index.name: index for index in self.indexes} + result = [] + for channel, names in selection.items(): + if not names or len(set(names)) != len(names): + raise ValueError("each channel requires distinct index names") # noqa: TRY003 + for name in names: + if name not in by_name or by_name[name].kind != channel: + raise ValueError("index is not declared for this channel") # noqa: TRY003 + result.append(by_name[name]) + profiles = [index.profile for index in result if isinstance(index, VectorIndex)] + if profiles and any(profile != profiles[0] for profile in profiles): + raise ValueError("selected vector indexes require the same complete EmbeddingProfile") # noqa: TRY003 + return tuple(result) + + +def validate_index_schema(index: TextIndex | VectorIndex, schema: dict[str, Any]) -> None: # noqa: C901 - Keep the complete projection or publication invariant together. + """Validate explicit required fields; no JSONPath or producer code is executed.""" + + def resolve(value): + seen = set() + while "$ref" in value: + ref = value["$ref"] + if not ref.startswith("#/$defs/") or ref in seen: + raise ValueError("index fields require local, nonrecursive schema definitions") # noqa: TRY003 + seen.add(ref) + value = schema["$defs"][ref.removeprefix("#/$defs/")] + return value + + def field(obj, name, kind): + obj = resolve(obj) + value = resolve(obj.get("properties", {}).get(name, {})) + if name not in obj.get("required", ()) or value.get("type") != kind: + raise ValueError(f"index field {name!r} must be a required {kind}") # noqa: TRY003 + return value + + obj = schema + if isinstance(index, VectorIndex) and index.fragments is not None: + array = field(obj, index.fragments.array_field, "array") + obj = resolve(array.get("items", {})) + if obj.get("type") != "object": + raise ValueError("fragments must be persistent objects") # noqa: TRY003 + field(obj, index.fragments.id_field, "string") + if isinstance(index, TextIndex) or index.source == "text": + for name in index.fields: + field(obj, name, "string") + else: + vector = field(obj, index.vector_field, "array") + if resolve(vector.get("items", {})).get("type") not in {"number", "integer"}: + raise ValueError("vectors must contain numbers") # noqa: TRY003 + field(obj, index.profile_field, "object") diff --git a/src/powercontext/artifacts/search.py b/src/powercontext/artifacts/search.py index a6e005f261..75d6471250 100644 --- a/src/powercontext/artifacts/search.py +++ b/src/powercontext/artifacts/search.py @@ -154,6 +154,15 @@ def __init__(self, family: str, *, field: str | None = None) -> None: super().__init__(f"Artifact search is unsupported for {family}" + (f": {field}" if field else "")) +class ArtifactSearchUnavailable(PowerContextError, RuntimeError): + """A declared search capability cannot serve a complete current projection.""" + + def __init__(self, family: str, reason: str) -> None: + self.family = family + self.reason = reason + super().__init__(f"Artifact search is unavailable for {family}: {reason}") + + class ArtifactSearchContractError(PowerContextError, RuntimeError): """A search implementation returned values that violate the public contract.""" @@ -179,6 +188,7 @@ def _require_finite_real(value: object, label: str) -> None: "ArtifactSearchMatch", "ArtifactSearchOutcome", "ArtifactSearchQuery", + "ArtifactSearchUnavailable", "ArtifactSearchUnsupported", "ArtifactSearcher", "ChannelScore", diff --git a/src/powercontext/builtin/artifact_plugin_validation.py b/src/powercontext/builtin/artifact_plugin_validation.py new file mode 100644 index 0000000000..cb0de0474e --- /dev/null +++ b/src/powercontext/builtin/artifact_plugin_validation.py @@ -0,0 +1,260 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Isolated Artifact declaration validation without starting the generation runtime.""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib.util +import inspect +import json +import re +import sys +from datetime import timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Annotated, Any + +import yaml +from jsonschema import Draft202012Validator +from pydantic import BaseModel, ConfigDict, Field, JsonValue, SecretBytes, SecretStr, field_validator + +from powercontext.artifact_plugins import InputContract +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, validate_index_schema +from powercontext.sources import SourceProjectionKey, SourceRef + +_MAX_INPUT_BYTES = 1024 * 1024 +_MAX_OUTPUT_BYTES = 1024 * 1024 +_BUILTIN_FAMILIES = frozenset({ + "memory", + "atomic-memory", + "experience", + "skill", + "handoff", + "profile", + "prompt", + "topic-memory", +}) + + +class _ProcessingConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True, allow_inf_nan=False) + + interval_seconds: float = Field(gt=0) + max_workers: int = Field(ge=1) + timeout_seconds: float = Field(gt=0) + + @field_validator("interval_seconds") + @classmethod + def representable_interval(cls, value: float) -> float: + try: + interval = timedelta(seconds=value) + except OverflowError as error: + raise ValueError("interval_seconds exceeds the supported duration") from error # noqa: TRY003 + if interval.total_seconds() <= 0: + raise ValueError("interval_seconds must be at least one microsecond") # noqa: TRY003 + return value + + +class _PluginManifest(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + id: str = Field(pattern=r"^[a-z][a-z0-9-]{0,127}$") + version: str = Field(min_length=1, max_length=128, pattern=r".*\S.*") + entry: str = Field(min_length=3, max_length=4096) + binding_name: str = Field(pattern=r"^[a-z][a-z0-9-]{0,127}$") + scope_ids: list[Annotated[str, Field(min_length=1, max_length=256)]] | None = Field( + default=None, min_length=1, max_length=1000 + ) + max_sources: int = Field(ge=1) + processing: _ProcessingConfig + config: dict[str, JsonValue] + + @field_validator("scope_ids") + @classmethod + def unique_scopes(cls, value: list[str] | None) -> list[str] | None: + if value is not None and len(set(value)) != len(value): + raise ValueError("Scope IDs must be unique") # noqa: TRY003 + return None if value is None else sorted(value) + + +def read_manifest(directory: str) -> _PluginManifest: + path = Path(directory) / "plugin.yml" + if path.stat().st_size > _MAX_INPUT_BYTES: + raise ValueError("plugin manifest is too large") # noqa: TRY003 + return _PluginManifest.model_validate(yaml.safe_load(path.read_text(encoding="utf-8"))) + + +def code_fingerprint(directory: str) -> str: + root = Path(directory) + digest = hashlib.sha256() + files = sorted( + path + for path in root.rglob("*") + if path.is_file() and not {"__pycache__", ".venv", ".git"}.intersection(path.relative_to(root).parts) + ) + for path in files: + digest.update(str(path.relative_to(root)).encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def load_plugin(directory: str, manifest: _PluginManifest): + root = Path(directory) + entry, factory = manifest.entry.rsplit(":", 1) + path = (root / entry).resolve() + if Path(entry).is_absolute() or not path.is_relative_to(root.resolve()) or path.suffix != ".py": + raise ValueError("entry must be a Python file inside the plugin directory") # noqa: TRY003 + module_name = "powercontext_user_artifact_plugin" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ValueError("plugin entry is unavailable") # noqa: TRY003 + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + sys.path.insert(0, str(root)) + spec.loader.exec_module(module) + module.__dict__["create_pipeline"] = getattr(module, factory) + return module + + +def describe_plugin(directory: str, snapshot: dict[str, Any] | None = None) -> dict[str, Any]: + manifest = read_manifest(directory) if snapshot is None else _PluginManifest.model_validate(snapshot) + return describe_module(load_plugin(directory, manifest), manifest, directory) + + +def _input_declaration(inputs: InputContract) -> dict[str, Any]: + if not isinstance(inputs, InputContract) or not isinstance(inputs.projections, dict) or not inputs.projections: + raise ValueError("inputs must declare a nonempty Source projection mapping") # noqa: TRY003 + projections = {} + for name, projection in inputs.projections.items(): + SourceRef(source_type=name, source_id="validation") + projections[name] = SourceProjectionKey.model_validate(projection).model_dump(mode="json") + families = _read_families(inputs.artifact_families) + return {"projections": projections, **({"artifact_families": families} if families else {})} + + +def _read_families(families): + if not isinstance(families, tuple): + raise TypeError("read families must be a tuple") # noqa: TRY003 + for family in families: + if not isinstance(family, str) or re.fullmatch(r"[a-z][a-z0-9-]{0,127}", family) is None or family == "memory": + raise ValueError("read families must name canonical Artifact families") # noqa: TRY003 + if len(set(families)) != len(families): + raise ValueError("read families must be distinct") # noqa: TRY003 + return sorted(families) + + +def _configuration_json(value, serialized): + # Config stays private to management/Workers. Preserve credentials instead of + # persisting SecretStr's display mask, while retaining JSON normalization. + if isinstance(value, SecretStr): + return value.get_secret_value() + if isinstance(value, SecretBytes): + return value.get_secret_value().decode("utf-8") + if isinstance(value, dict) and isinstance(serialized, dict): + return {key: _configuration_json(value[key], item) for key, item in serialized.items()} + if isinstance(value, (list, tuple)) and isinstance(serialized, list): + return [_configuration_json(original, item) for original, item in zip(value, serialized, strict=True)] + return serialized + + +def _validate_factory(factory) -> None: + if not inspect.iscoroutinefunction(factory): + raise ValueError("plugin factory must be async") # noqa: TRY003 + try: + inspect.signature(factory).bind(None, None) + except (TypeError, ValueError) as error: + raise ValueError("plugin factory must accept context and configuration") from error # noqa: TRY003 + + +def describe_module(module, manifest: _PluginManifest, directory: str) -> dict[str, Any]: + if module.plugin_api_version != "1" or len(module.families) != 1: + raise ValueError("API version 1 requires exactly one output family") # noqa: TRY003 + _validate_factory(module.create_pipeline) + family = module.families[0] + if re.fullmatch(r"[a-z][a-z0-9-]{0,127}", family.family) is None or family.family in _BUILTIN_FAMILIES: + raise ValueError("a plugin needs a custom canonical family") # noqa: TRY003 + if not family.schema_version or len(family.schema_version) > 128: + raise ValueError("a bounded schema_version is required") # noqa: TRY003 + schema = family.content_model.model_json_schema() + Draft202012Validator.check_schema(schema) + if schema.get("type") != "object": + raise ValueError("Artifact content must be a JSON object") # noqa: TRY003 + fields = () + search = family.search + if search is not None: + search = ArtifactSearchDeclaration.model_validate(search) + for index in search.indexes: + validate_index_schema(index, schema) + # JSON validation restores strict Config types from the serialized definition snapshot. + validated_config = module.Config.model_validate_json(json.dumps(manifest.config, ensure_ascii=False)) + snapshot = manifest.model_dump(mode="json") + snapshot["config"] = _configuration_json( + validated_config.model_dump(mode="python", by_alias=True), + validated_config.model_dump(mode="json", by_alias=True), + ) + return { + "manifest": snapshot, + "declaration": { + "plugin_id": manifest.id, + "version": manifest.version, + "family": family.family, + "schema_version": family.schema_version, + "content_schema": schema, + "search_fields": list(fields), + **({"search": search.model_dump(mode="json")} if search is not None else {}), + **_input_declaration(module.inputs), + "fingerprint": code_fingerprint(directory), + }, + } + + +async def validate_plugin(directory: str, snapshot: dict[str, Any] | None = None) -> dict[str, Any]: + # Result files keep user import-time stdout out of the declaration protocol. + with TemporaryDirectory(prefix="powercontext-artifact-validation-") as temporary: + root = Path(temporary) + (root / "manifest.json").write_text(json.dumps(snapshot, ensure_ascii=False), encoding="utf-8") + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + __name__, + directory, + str(root / "manifest.json"), + str(root / "result.json"), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + async with asyncio.timeout(15): + await process.wait() + except BaseException: + if process.returncode is None: + process.kill() + await process.wait() + raise + if process.returncode: + raise ValueError("plugin declaration or configuration validation failed") # noqa: TRY003 + output = root / "result.json" + if output.stat().st_size > _MAX_OUTPUT_BYTES: + raise ValueError("plugin declaration is too large") # noqa: TRY003 + return json.loads(output.read_text()) + + +if __name__ == "__main__": + result = describe_plugin(sys.argv[1], json.loads(Path(sys.argv[2]).read_text())) + Path(sys.argv[3]).write_text(json.dumps(result, ensure_ascii=False)) diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 94ddb3abd0..cd4b6bddbc 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -34,6 +34,9 @@ if TYPE_CHECKING: import jieba +# Bump when document segmentation or token encoding changes; query stopwords do not affect projections. +TEXT_ANALYZER_VERSION = "jieba-0.42.1-words-v1" + _FTS_MIN_QUERY_COVERAGE = 0.25 _FTS_MIN_MATCHED_TERMS = 2 _FTS_SHORT_QUERY_MAX_TERMS = 2 @@ -418,6 +421,7 @@ def admits_fts_projection(query: str, projection: str, /, *, floor: AdmissionFlo __all__ = [ "DEFAULT_ADMISSION_FLOOR", + "TEXT_ANALYZER_VERSION", "AdmissionCounts", "AdmissionFloor", "EmptyFilters", diff --git a/src/powercontext/builtin/persistence/artifact_governance.py b/src/powercontext/builtin/persistence/artifact_governance.py index a4aadc36ba..8d5b11c594 100644 --- a/src/powercontext/builtin/persistence/artifact_governance.py +++ b/src/powercontext/builtin/persistence/artifact_governance.py @@ -17,6 +17,7 @@ from __future__ import annotations from enum import StrEnum +from typing import get_args from pydantic import BaseModel, ConfigDict, Field, model_validator from sqlalchemy import select, update @@ -25,6 +26,7 @@ from powercontext.artifacts import ArtifactRef from powercontext.builtin.persistence.errors import RepositoryNotFoundError, StoredPayloadConflictError from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE +from powercontext.builtin.records import BaseArtifactFamily from powercontext.errors import PowerContextError @@ -67,6 +69,30 @@ def validate_replacement(self) -> ArtifactGovernance: class ArtifactGovernanceRepository: """Read and transition governance state without changing the head Revision.""" + def __init__(self, *, prepared_indexes=None): + self.prepared_indexes = prepared_indexes or {} + + async def _prepare_custom_transition(self, connection, scope_id, family, artifact_id, lifecycle_state): + if family in get_args(BaseArtifactFamily): + return None + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher, lock_state + from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository + + await lock_state(connection, scope_id, family) + artifact = await custom_artifact_repository(family).latest(connection, scope_id, family, artifact_id) + publisher = ArtifactProjectionPublisher(None) + prepared = self.prepared_indexes.get((scope_id, family, artifact_id)) + if lifecycle_state is ArtifactLifecycleState.ACTIVE: + await publisher.validate_prepared( + connection, scope_id, family, artifact.content.schema_version, artifact.content.content, prepared + ) + return publisher, artifact, prepared + + async def _publish_custom_transition(self, connection, scope_id, custom): + if custom is not None: + publisher, artifact, prepared = custom + await publisher.publish(connection, scope_id, artifact, prepared) + async def get( self, connection: AsyncConnection, @@ -115,6 +141,7 @@ async def transition_merge( ) -> ArtifactGovernance: """Apply a service-validated group transition while retaining unchanged state generations.""" + custom = await self._prepare_custom_transition(connection, scope_id, family, artifact_id, lifecycle_state) _validate_transition(expected.lifecycle_state, lifecycle_state) current = await self.require_summary(connection, scope_id, family, artifact_id, expected) if ( @@ -147,6 +174,7 @@ async def transition_merge( ) if result.rowcount != 1: raise StoredPayloadConflictError("artifact-governance", (scope_id, family, artifact_id)) + await self._publish_custom_transition(connection, scope_id, custom) return requested async def transition( @@ -160,6 +188,7 @@ async def transition( replacement_artifact_id: str | None, /, ) -> ArtifactGovernance: + custom = await self._prepare_custom_transition(connection, scope_id, family, artifact_id, lifecycle_state) current = await self.get(connection, scope_id, family, artifact_id) if current.merged_into_id is not None: raise InvalidArtifactLifecycleError("frozen merge inputs require whole-group restoration") # noqa: TRY003 @@ -206,6 +235,7 @@ async def transition( raise StoredPayloadConflictError( "artifact-governance", (scope_id, family, artifact_id, expected_generation) ) + await self._publish_custom_transition(connection, scope_id, custom) return requested diff --git a/src/powercontext/builtin/persistence/artifact_indexes.py b/src/powercontext/builtin/persistence/artifact_indexes.py new file mode 100644 index 0000000000..1599f615a9 --- /dev/null +++ b/src/powercontext/builtin/persistence/artifact_indexes.py @@ -0,0 +1,1053 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Transactional custom Artifact projections, build coordination and rebuilds.""" + +from __future__ import annotations + +import json +import math +import struct +from copy import deepcopy +from dataclasses import dataclass +from hashlib import sha256 +from typing import Any +from uuid import uuid4 + +from sqlalchemy import delete, insert, select, text, update + +from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchFamilyNotFound, ArtifactSearchUnavailable +from powercontext.builtin.artifacts.memory.canonical import canonical_embedding, canonical_json +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.artifacts.search import TEXT_ANALYZER_VERSION, analyze_text +from powercontext.builtin.persistence.sqlite.profile import ensure_sqlite_vector_extension +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE as SCHEMAS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE as HEADS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_INDEX_CHANGES_TABLE as CHANGES, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_INDEX_COVERAGE_TABLE as COVERAGE, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_SEARCH_STATES_TABLE as STATES, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_SEARCH_TEXT_TABLE as TEXTS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTOR_INDEXES_TABLE as DEFINITIONS, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTOR_SPACES_TABLE as SPACES, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTORS_TABLE as VECTORS, +) + +FORMAT = "float32-le-v1" + + +def identity_json(value): + # IDs, raw content and fragment locators must not be Unicode-normalized by hashing. + return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode() + + +def digest(value): + return sha256(identity_json(value)).hexdigest() + + +def search_declaration(durable): + if durable.get("search") is not None: + return ArtifactSearchDeclaration.model_validate(durable["search"]) + fields = durable.get("search_fields", []) + if fields: + return ArtifactSearchDeclaration( + version="legacy-fts-v1", + indexes=(TextIndex(name="lexical", projection_version="1", fields=tuple(fields)),), + modes=("text",), + default_mode="text", + ) + return None + + +def key(table, scope, family): + return (table.c.scope_id == scope) & (table.c.family == family) + + +async def lock_state(connection, scope, family): + values = { + "scope_id": scope, + "family": family, + "build_id": None, + "state": "pending", + "manifest": {}, + "cleanup_targets": [], + "start_seq": 0, + "applied_seq": 0, + "change_seq": 0, + } + if connection.dialect.name == "sqlite": + from sqlalchemy.dialects.sqlite import insert as dialect_insert + + stmt = dialect_insert(STATES).values(**values).on_conflict_do_nothing(index_elements=["scope_id", "family"]) + else: + from sqlalchemy.dialects.mysql import insert as dialect_insert + + stmt = dialect_insert(STATES).values(**values).on_duplicate_key_update(family=family) + await connection.execute(stmt) + # This is a real write on SQLite, and the same row lock as SELECT FOR UPDATE on MySQL. + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(change_seq=STATES.c.change_seq)) + return ( + (await connection.execute(select(STATES).where(key(STATES, scope, family)).with_for_update())).mappings().one() + ) + + +async def read_head(connection, scope, family, artifact_id, *, for_update=False): + statement = select(HEADS).where(key(HEADS, scope, family), HEADS.c.artifact_id == artifact_id) + if for_update: + statement = statement.with_for_update() + return (await connection.execute(statement)).mappings().one_or_none() + + +async def record_change(connection, scope, family, artifact_id, schema_version, operation): + state = await lock_state(connection, scope, family) + head = await read_head(connection, scope, family, artifact_id) + sequence = state["change_seq"] + 1 + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(change_seq=sequence)) + await connection.execute( + insert(CHANGES).values( + scope_id=scope, + family=family, + seq=sequence, + artifact_id=artifact_id, + revision=None if head is None else head["revision"], + schema_version=schema_version, + governance_generation=0 if head is None else head["governance_generation"], + operation=operation, + ) + ) + if state["state"] == "ready": + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(applied_seq=sequence)) + await prune_changes(connection, scope, family) + return sequence + + +async def prune_changes(connection, scope, family): + applied = await connection.scalar(select(STATES.c.applied_seq).where(key(STATES, scope, family))) + await connection.execute(delete(CHANGES).where(key(CHANGES, scope, family), CHANGES.c.seq <= applied)) + + +async def register_definition(connection, family, version, index): + spec = index.model_dump(mode="json") + if isinstance(index, TextIndex): + spec["analyzer_version"] = TEXT_ANALYZER_VERSION + space = None + if isinstance(index, VectorIndex): + fingerprint = sha256(canonical_json(index.profile.model_dump(mode="json"))).hexdigest() + backend = connection.dialect.name + space = uuid4().hex + values = { + "space_id": space, + "profile_fingerprint": fingerprint, + "profile_json": index.profile.model_dump(mode="json"), + "dimension": index.profile.dimension, + "distance": index.profile.distance, + "normalization": index.profile.normalization, + "storage_format_version": FORMAT, + "backend": backend, + "physical_table": "pc_artifact_vectors" if backend == "sqlite" else f"pc_av_{space}", + "provision_state": "ready" if backend == "sqlite" else "declared", + } + if backend == "sqlite": + from sqlalchemy.dialects.sqlite import insert as space_insert + + statement = ( + space_insert(SPACES) + .values(**values) + .on_conflict_do_nothing(index_elements=["backend", "profile_fingerprint", "storage_format_version"]) + ) + else: + from sqlalchemy.dialects.mysql import insert as space_insert + + statement = space_insert(SPACES).values(**values).on_duplicate_key_update(backend=backend) + await connection.execute(statement) + space_row = ( + ( + await connection.execute( + select(SPACES) + .where( + SPACES.c.backend == backend, + SPACES.c.profile_fingerprint == fingerprint, + SPACES.c.storage_format_version == FORMAT, + ) + .with_for_update() + ) + ) + .mappings() + .one() + ) + if space_row["profile_json"] != index.profile.model_dump(mode="json"): + raise ValueError("vector space fingerprint collision") # noqa: TRY003 + space = space_row["space_id"] + identity = { + "family": family, + "schema_version": version, + "index_name": index.name, + "projection_version": index.projection_version, + "space_id": space, + } + existing = ( + ( + await connection.execute( + select(DEFINITIONS) + .where(*(DEFINITIONS.c[name] == value for name, value in identity.items())) + .order_by(DEFINITIONS.c.definition_fingerprint) + ) + ) + .mappings() + .first() + ) + fingerprint = digest(spec) + exact = ( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == digest({**identity, "spec": spec})) + ) + ) + .mappings() + .first() + ) + if exact is not None: + if any(exact[name] != value for name, value in identity.items()) or exact["projection_spec"] != spec: + raise ValueError("index definition identity collision") # noqa: TRY003 + return dict(exact) + if existing is not None: + prior_spec = {k: v for k, v in existing["projection_spec"].items() if k != "analyzer_version"} + current_spec = {k: v for k, v in spec.items() if k != "analyzer_version"} + if prior_spec != current_spec: + raise ValueError("index projection version cannot be redefined") # noqa: TRY003 + if existing["definition_fingerprint"] == fingerprint: + return dict(existing) + row = dict( + index_def_id=digest({**identity, "spec": spec}), + **identity, + source_kind="text" if isinstance(index, TextIndex) else index.source, + projection_spec=spec, + definition_fingerprint=fingerprint, + ) + await connection.execute(insert(DEFINITIONS).values(**row)) + return row + + +def parse_index(definition): + spec = {k: v for k, v in definition["projection_spec"].items() if k != "analyzer_version"} + return (TextIndex if spec["kind"] == "text" else VectorIndex).model_validate(spec) + + +async def target_manifest(connection, scope, family, *, rebuild=False): # noqa: C901 - Initial build and repair share one fenced state transition. + """Initialize a fixed target or restart repair from its persisted declaration.""" + state = await lock_state(connection, scope, family) + manifest = deepcopy(state["manifest"]) + if manifest: + if not rebuild: + return manifest + for _, entry in entries(manifest): + entry.update(scan_cursor=None, scan_complete=False) + else: + rows = ( + ( + await connection.execute( + select(SCHEMAS).where(SCHEMAS.c.family == family).order_by(SCHEMAS.c.schema_version) + ) + ) + .mappings() + .all() + ) + if not rows: + raise ArtifactSearchFamilyNotFound(family) + manifest = {"schemas": {}, "config": None, "default_schema": None} + for row in rows: + declaration = search_declaration(row["declaration"]) + if declaration is None: + continue + schema = row["schema_version"] + if manifest["config"] is None: + manifest["config"] = declaration.model_dump(mode="json") + manifest["default_schema"] = schema + indexes = {} + for index in declaration.indexes: + definition = await register_definition(connection, family, schema, index) + indexes[index.name] = { + "index_def_id": definition["index_def_id"], + "space_id": definition["space_id"], + "scan_cursor": None, + "scan_complete": False, + } + manifest["schemas"][schema] = {"indexes": indexes} + targets = list(state["cleanup_targets"]) + if state["build_id"] is not None: + for _, entry in entries(state["manifest"]): + target = { + "build_id": state["build_id"], + "index_def_id": entry["index_def_id"], + "space_id": entry["space_id"], + } + if target not in targets: + targets.append(target) + await connection.execute( + update(STATES) + .where(key(STATES, scope, family)) + .values( + build_id=uuid4().hex, + state="cleaning" if targets else "building", + manifest=manifest, + cleanup_targets=targets, + validation_token=None, + start_seq=state["change_seq"], + applied_seq=state["change_seq"], + ) + ) + await prune_changes(connection, scope, family) + return manifest + + +def entries(manifest): + return tuple( + (schema, entry) + for schema, config in manifest.get("schemas", {}).items() + for entry in config["indexes"].values() + ) + + +def vector_bytes(values, profile): + if any(isinstance(v, bool) or not isinstance(v, (int, float)) for v in values): + raise ValueError("vectors must contain finite numbers") # noqa: TRY003 + vector = canonical_embedding(values, dimension=profile.dimension, normalization=profile.normalization) + try: + encoded = struct.pack(f"<{len(vector)}f", *vector) + except (OverflowError, struct.error) as error: + raise ValueError("vector exceeds float32 range") from error # noqa: TRY003 + if not all(math.isfinite(v) for v in struct.unpack(f"<{len(vector)}f", encoded)): + raise ValueError("vector exceeds float32 range") # noqa: TRY003 + return encoded + + +def index_parts(index, content): + if isinstance(index, VectorIndex) and index.fragments is not None: + parts = content[index.fragments.array_field] + if not isinstance(parts, list): + raise ValueError("fragment projection requires a persistent array") # noqa: TRY003 + else: + parts = [content] + seen = set() + for offset, part in enumerate(parts): + fragment_id = "__whole__" + locator = {"fields": list(index.fields)} + if isinstance(index, VectorIndex) and index.fragments is not None: + fragment_id = part[index.fragments.id_field] + if not isinstance(fragment_id, str) or not 1 <= len(fragment_id) <= 128 or fragment_id == "__whole__": + raise ValueError("fragment identity must be a bounded persistent string") # noqa: TRY003 + locator = {"array": index.fragments.array_field, "id": fragment_id, "offset": offset} + if fragment_id in seen: + raise ValueError("fragment identities must be unique") # noqa: TRY003 + seen.add(fragment_id) + yield part, fragment_id, locator + + +def content_vector(index, part): + if EmbeddingProfile.model_validate(part[index.profile_field]) != index.profile: + raise ValueError("persistent vector profile differs from the declared index") # noqa: TRY003 + return vector_bytes(part[index.vector_field], index.profile) + + +@dataclass(frozen=True) +class PreparedIndex: + definition: dict[str, Any] + fragments: tuple[dict[str, Any], ...] + input_hash: str + + +@dataclass(frozen=True) +class PreparedArtifactIndexes: + build_id: str | None + content_hash: str + projections: dict[str, PreparedIndex] + + +class ArtifactProjectionPublisher: + """Prepare without a transaction; publish only under the build coordinator.""" + + def __init__(self, database, embedding_model=None): + self.database = database + self.embedding_model = embedding_model + + async def prepare_index(self, definition, content): + index = parse_index(definition) + fragments = [] + texts = [] + for part, fragment_id, locator in index_parts(index, content): + value = {"fragment_id": fragment_id, "fragment_locator": identity_json(locator).decode()} + if isinstance(index, TextIndex) or index.source == "text": + fields = [part[field] for field in index.fields] + if any(not isinstance(field, str) for field in fields): + raise ValueError("text projection requires strings") # noqa: TRY003 + original = "\n".join(fields) + if isinstance(index, TextIndex): + value["searchable_text"] = analyze_text(original) + else: + texts.append(original) + else: + value["embedding"] = content_vector(index, part) + value["input_hash"] = digest({"projection": definition["definition_fingerprint"], "part": part}) + fragments.append(value) + if texts: + model = self.embedding_model + if model is None or model.profile != index.profile: + raise ArtifactSearchUnavailable( + definition["family"], "embedding resource unavailable for declared profile" + ) + result = await model.embed(tuple(texts)) + if len(result.vectors) != len(texts): + raise ValueError("embedding result must match the prepared input count") # noqa: TRY003 + for fragment, vector in zip(fragments, result.vectors, strict=True): + fragment["embedding"] = vector_bytes(vector, index.profile) + return PreparedIndex( + definition, + tuple(fragments), + digest({"projection": definition["definition_fingerprint"], "content": content}), + ) + + async def prepare(self, scope, family, schema_version, content): + content = deepcopy(content) + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().first() + if state is None or state["state"] != "ready": + return PreparedArtifactIndexes(None, digest(content), {}) + definitions = {} + for schema, entry in entries(state["manifest"]): + if schema == schema_version: + definitions[entry["index_def_id"]] = dict( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == entry["index_def_id"]) + ) + ) + .mappings() + .one() + ) + projections = {} + try: + for identity, definition in definitions.items(): + projections[identity] = await self.prepare_index(definition, content) + except Exception: + # A rebuild may have suspended this index while encoding was in flight. + # Content writes then require only the durable change log. + async with self.database.transaction(consistent_snapshot=True) as connection: + current = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if current["state"] == "ready": + raise + return PreparedArtifactIndexes(None, digest(content), {}) + return PreparedArtifactIndexes(state["build_id"], digest(content), projections) + + async def validate_prepared(self, connection, scope, family, schema_version, content, prepared): + state = await lock_state(connection, scope, family) + if state["state"] != "ready": + durable = await connection.scalar( + select(SCHEMAS.c.declaration).where( + SCHEMAS.c.family == family, SCHEMAS.c.schema_version == schema_version + ) + ) + declaration = None if durable is None else search_declaration(durable) + if declaration is not None: + # Resource-independent content invariants hold even while projections are suspended. + for index in declaration.indexes: + if isinstance(index, VectorIndex): + for part, _, _ in index_parts(index, content): + if index.source == "content": + content_vector(index, part) + return state + expected = {entry["index_def_id"] for schema, entry in entries(state["manifest"]) if schema == schema_version} + if prepared is None: + if expected: + raise ValueError("ready Artifact indexes must be prepared before the write transaction") # noqa: TRY003 + elif ( + prepared.build_id != state["build_id"] + or prepared.content_hash != digest(content) + or set(prepared.projections) != expected + ): + raise ValueError("Artifact content or search build changed; prepare again before committing") # noqa: TRY003 + return state + + async def publish(self, connection, scope, artifact, prepared): + family, identity = artifact.family, artifact.artifact_id + state = await lock_state(connection, scope, family) + head = await read_head(connection, scope, family, identity, for_update=True) + if state["state"] == "ready": + if head["lifecycle_state"] == "active": + await self.validate_prepared( + connection, scope, family, artifact.content.schema_version, artifact.content.content, prepared + ) + for _, entry in entries(state["manifest"]): + projection = ( + None + if prepared is None or head["lifecycle_state"] != "active" + else prepared.projections.get(entry["index_def_id"]) + ) + await self.replace( + connection, + scope, + family, + identity, + artifact.revision, + head["governance_generation"], + state["build_id"], + entry["index_def_id"], + projection, + ) + await record_change(connection, scope, family, identity, artifact.content.schema_version, "upsert") + + async def replace( + self, connection, scope, family, artifact_id, revision, governance, build_id, index_def_id, prepared + ): + state = await lock_state(connection, scope, family) + if ( + state["build_id"] != build_id + or state["state"] not in {"ready", "building"} + or state["validation_token"] is not None + ): + raise ValueError("projection build is not writable") # noqa: TRY003 + if index_def_id not in {entry["index_def_id"] for _, entry in entries(state["manifest"])}: + raise ValueError("projection definition is not in the current build") # noqa: TRY003 + definition = dict( + (await connection.execute(select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == index_def_id))) + .mappings() + .one() + ) + index = parse_index(definition) + table = await projection_table(connection, definition) + await connection.execute( + delete(table).where( + projection_key(table, scope, family, build_id, index_def_id), table.c.artifact_id == artifact_id + ) + ) + await connection.execute( + delete(COVERAGE).where( + projection_key(COVERAGE, scope, family, build_id, index_def_id), COVERAGE.c.artifact_id == artifact_id + ) + ) + if prepared is None: + return + if prepared.definition != definition: + raise ValueError("prepared definition does not match its index") # noqa: TRY003 + schema = definition["schema_version"] + rows = [] + for fragment in prepared.fragments: + identity = ( + scope, + family, + artifact_id, + revision, + schema, + definition["index_def_id"], + build_id, + fragment["fragment_id"], + ) + row = dict( + scope_id=scope, + family=family, + artifact_id=artifact_id, + revision=revision, + schema_version=schema, + index_def_id=definition["index_def_id"], + build_id=build_id, + governance_generation=governance, + **fragment, + ) + if isinstance(index, VectorIndex): + row.update( + vector_id=digest(identity), space_id=definition["space_id"], dimension=index.profile.dimension + ) + if connection.dialect.name != "sqlite": + row["embedding"] = list(struct.unpack(f"<{index.profile.dimension}f", row["embedding"])) + else: + row["text_id"] = digest(identity) + rows.append(row) + if rows: + await connection.execute(insert(table), rows) + await connection.execute( + insert(COVERAGE).values( + scope_id=scope, + family=family, + index_def_id=index_def_id, + build_id=build_id, + artifact_id=artifact_id, + revision=revision, + schema_version=schema, + fragment_count=len(rows), + projection_input_hash=prepared.input_hash, + governance_generation=governance, + ) + ) + + +def projection_key(table, scope, family, build_id, index_def_id): + return key(table, scope, family) & (table.c.build_id == build_id) & (table.c.index_def_id == index_def_id) + + +async def projection_table(connection, definition): + if definition["space_id"] is None: + return TEXTS + space = ( + (await connection.execute(select(SPACES).where(SPACES.c.space_id == definition["space_id"]))).mappings().one() + ) + if space["provision_state"] != "ready": + raise ArtifactSearchUnavailable(definition["family"], "vector space preparation is incomplete") + if connection.dialect.name == "sqlite": + return VECTORS + from powercontext.builtin.persistence.artifact_vector_space import native_vector_table + + try: + return native_vector_table(space) + except ValueError as error: + raise ArtifactSearchUnavailable(definition["family"], "registered vector space is incompatible") from error + + +class ArtifactIndexBuilder: + """Resume cleanup, scans and replay while the complete family search is unavailable.""" + + def __init__(self, database, embedding_model=None): + self.database = database + self.publisher = ArtifactProjectionPublisher(database, embedding_model) + + async def refresh(self, scope, family, *, rebuild=False): + if not rebuild: + async with self.database.transaction(consistent_snapshot=True) as connection: + manifest = await connection.scalar(select(STATES.c.manifest).where(key(STATES, scope, family))) + if manifest: + return manifest + async with self.database.transaction() as connection: + return await target_manifest(connection, scope, family, rebuild=rebuild) + + async def clean(self, scope, family, build_id, *, limit=1000): + """Delete a bounded batch, fenced by the current build and its coordinator lock.""" + async with self.database.transaction() as connection: + state = await lock_state(connection, scope, family) + if state["build_id"] != build_id or state["state"] != "cleaning": + return False + targets = list(state["cleanup_targets"]) + remaining = limit + while targets and remaining: + target = targets[0] + definition = ( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == target["index_def_id"]) + ) + ) + .mappings() + .one() + ) + tables = [COVERAGE] + # An unprovisioned space has never accepted projection rows. + if ( + target["space_id"] is None + or await connection.scalar( + select(SPACES.c.provision_state).where(SPACES.c.space_id == target["space_id"]) + ) + == "ready" + ): + tables.insert(0, await projection_table(connection, definition)) + complete = True + for table in tables: + where = projection_key(table, scope, family, target["build_id"], target["index_def_id"]) + primary = table.c.artifact_id if table is COVERAGE else next(iter(table.primary_key.columns)) + ids = (await connection.execute(select(primary).where(where).limit(remaining))).scalars().all() + if ids: + await connection.execute(delete(table).where(where, primary.in_(ids))) + remaining -= len(ids) + if not remaining: + complete = False + break + if complete: + targets.pop(0) + await connection.execute( + update(STATES) + .where(key(STATES, scope, family)) + .values(cleanup_targets=targets, state="cleaning" if targets else "building") + ) + return not targets + + async def build(self, scope, family): # noqa: C901 - Resume ordered cleanup, scans, replay and validation. + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if state["state"] == "ready": + return True + if not state["manifest"]: + return False + build_id = state["build_id"] + if state["state"] == "cleaning" and not await self.clean(scope, family, build_id): + return False + # Reclaim an interrupted validator. Any previous validator must still present + # its own token at commit, so it cannot publish after this recovery. + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if current["build_id"] != build_id or current["state"] != "building": + return False + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(validation_token=None)) + manifest = current["manifest"] + for _, entry in entries(manifest): + if entry["scan_complete"]: + continue + cursor = entry["scan_cursor"] or "" + while True: + async with self.database.transaction(consistent_snapshot=True) as connection: + identities = ( + ( + await connection.execute( + select(HEADS.c.artifact_id) + .where(key(HEADS, scope, family), HEADS.c.artifact_id > cursor) + .order_by(HEADS.c.artifact_id) + .limit(100) + ) + ) + .scalars() + .all() + ) + if not identities: + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if not self._writable(current, build_id): + return False + progress = deepcopy(current["manifest"]) + for _, item in entries(progress): + if item["index_def_id"] == entry["index_def_id"]: + item["scan_complete"] = True + await connection.execute( + update(STATES).where(key(STATES, scope, family)).values(manifest=progress) + ) + break + for identity in identities: + if not await self._project_current( + scope, family, build_id, entry["index_def_id"], identity, cursor=identity + ): + return False + cursor = identity + for _ in range(8): + if not await self._catch_up(scope, family, build_id): + return False + if await self._validate_and_publish(scope, family, build_id): + return True + return False + + @staticmethod + def _writable(state, build_id): + return state["build_id"] == build_id and state["state"] == "building" and state["validation_token"] is None + + async def _project_current(self, scope, family, build_id, index_def_id, identity, *, cursor=None): + from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository + + repository = custom_artifact_repository(family) + for _ in range(8): + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if not self._writable(state, build_id): + return False + definition = dict( + (await connection.execute(select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == index_def_id))) + .mappings() + .one() + ) + head = await read_head(connection, scope, family, identity) + artifact = None + if head is not None and head["lifecycle_state"] == "active": + artifact = await repository.get( + connection, scope, ArtifactRef(family=family, artifact_id=identity, revision=head["revision"]) + ) + prepared = None + if artifact is not None and artifact.content.schema_version == definition["schema_version"]: + prepared = await self.publisher.prepare_index(definition, artifact.content.content) + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if not self._writable(current, build_id): + return False + if await read_head(connection, scope, family, identity, for_update=True) != head: + continue + await self.publisher.replace( + connection, + scope, + family, + identity, + 0 if head is None else head["revision"], + 0 if head is None else head["governance_generation"], + build_id, + index_def_id, + prepared, + ) + if cursor is not None: + progress = deepcopy(current["manifest"]) + for _, entry in entries(progress): + if entry["index_def_id"] == index_def_id: + entry["scan_cursor"] = max(cursor, entry["scan_cursor"] or "") + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(manifest=progress)) + return True + return False + + async def _catch_up(self, scope, family, build_id): + while True: + async with self.database.transaction(consistent_snapshot=True) as connection: + state = (await connection.execute(select(STATES).where(key(STATES, scope, family)))).mappings().one() + if not self._writable(state, build_id): + return False + changes = ( + ( + await connection.execute( + select(CHANGES) + .where(key(CHANGES, scope, family), CHANGES.c.seq > state["applied_seq"]) + .order_by(CHANGES.c.seq) + .limit(100) + ) + ) + .mappings() + .all() + ) + if not changes: + return True + for change in changes: + for _, entry in entries(state["manifest"]): + if not await self._project_current( + scope, family, build_id, entry["index_def_id"], change["artifact_id"] + ): + return False + async with self.database.transaction() as connection: + current = await lock_state(connection, scope, family) + if not self._writable(current, build_id): + return False + await connection.execute( + update(STATES) + .where(key(STATES, scope, family)) + .values(applied_seq=max(change["seq"], current["applied_seq"])) + ) + await prune_changes(connection, scope, family) + + async def _validate_and_publish(self, scope, family, build_id): + token = uuid4().hex + async with self.database.transaction() as connection: + state = await lock_state(connection, scope, family) + if not self._writable(state, build_id) or state["applied_seq"] != state["change_seq"]: + return False + if not all(entry["scan_complete"] for _, entry in entries(state["manifest"])): + return False + manifest, sequence = state["manifest"], state["change_seq"] + await connection.execute(update(STATES).where(key(STATES, scope, family)).values(validation_token=token)) + try: + async with self.database.transaction(consistent_snapshot=True) as connection: + valid = await self._coverage_valid(connection, scope, family, build_id, manifest) + if not valid: + return False + async with self.database.transaction() as connection: + state = await lock_state(connection, scope, family) + if ( + state["build_id"] != build_id + or state["state"] != "building" + or state["validation_token"] != token + or state["change_seq"] != sequence + or state["manifest"] != manifest + ): + return False + await connection.execute( + update(STATES).where(key(STATES, scope, family)).values(state="ready", validation_token=None) + ) + await prune_changes(connection, scope, family) + return True + finally: + async with self.database.transaction() as connection: + await lock_state(connection, scope, family) + await connection.execute( + update(STATES) + .where( + key(STATES, scope, family), STATES.c.build_id == build_id, STATES.c.validation_token == token + ) + .values(validation_token=None) + ) + + async def _coverage_valid(self, connection, scope, family, build_id, manifest): # noqa: C901 - Keep the complete projection or publication invariant together. + from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository + + repository = custom_artifact_repository(family) + heads = ( + ( + await connection.execute( + select(HEADS).where(key(HEADS, scope, family), HEADS.c.lifecycle_state == "active") + ) + ) + .mappings() + .all() + ) + artifacts = {} + for head in heads: + artifact = await repository.get( + connection, + scope, + ArtifactRef(family=family, artifact_id=head["artifact_id"], revision=head["revision"]), + ) + artifacts[head["artifact_id"]] = (head, artifact) + for schema, entry in entries(manifest): + definition = dict( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == entry["index_def_id"]) + ) + ) + .mappings() + .one() + ) + table = await projection_table(connection, definition) + index = parse_index(definition) + if isinstance(index, VectorIndex): + probe = vector_bytes([1.0] * index.profile.dimension, index.profile) + if connection.dialect.name == "sqlite": + await ensure_sqlite_vector_extension(connection) + await connection.execute(text("SELECT vec_distance_l2(:vector, :vector)"), {"vector": probe}) + else: + from sqlalchemy import bindparam + + values = list(struct.unpack(f"<{index.profile.dimension}f", probe)) + await connection.execute( + text("SELECT l2_distance(:probe, :probe)").bindparams( + bindparam("probe", type_=table.c.embedding.type) + ), + {"probe": values}, + ) + coverage = { + row["artifact_id"]: row + for row in ( + await connection.execute( + select(COVERAGE).where(projection_key(COVERAGE, scope, family, build_id, entry["index_def_id"])) + ) + ).mappings() + } + expected = { + identity for identity, (_, artifact) in artifacts.items() if artifact.content.schema_version == schema + } + if set(coverage) != expected: + return False + rows = ( + ( + await connection.execute( + select(table).where(projection_key(table, scope, family, build_id, entry["index_def_id"])) + ) + ) + .mappings() + .all() + ) + by_artifact = {} + for row in rows: + if row["artifact_id"] not in expected: + return False + by_artifact.setdefault(row["artifact_id"], []).append(row) + for identity in expected: + head, artifact = artifacts[identity] + content = artifact.content.content + row = coverage[identity] + parts = ( + content[index.fragments.array_field] + if isinstance(index, VectorIndex) and index.fragments + else [content] + ) + if ( + row["revision"], + row["schema_version"], + row["governance_generation"], + row["fragment_count"], + row["projection_input_hash"], + ) != ( + head["revision"], + schema, + head["governance_generation"], + len(parts), + digest({"projection": definition["definition_fingerprint"], "content": content}), + ): + return False + projected = by_artifact.get(identity, []) + if len(projected) != len(parts): + return False + expected_parts = { + ( + part[index.fragments.id_field] + if isinstance(index, VectorIndex) and index.fragments + else "__whole__" + ): (offset, part) + for offset, part in enumerate(parts) + } + if {item["fragment_id"] for item in projected} != set(expected_parts): + return False + for item in projected: + offset, part = expected_parts[item["fragment_id"]] + locator = ( + {"array": index.fragments.array_field, "id": item["fragment_id"], "offset": offset} + if isinstance(index, VectorIndex) and index.fragments + else {"fields": list(index.fields)} + ) + if ( + item["input_hash"] != digest({"projection": definition["definition_fingerprint"], "part": part}) + or item["fragment_locator"] != identity_json(locator).decode() + ): + return False + if isinstance(index, TextIndex): + if item["searchable_text"] != analyze_text("\n".join(part[field] for field in index.fields)): + return False + else: + if (item["space_id"], item["dimension"]) != (definition["space_id"], index.profile.dimension): + return False + vector = ( + item["embedding"] + if connection.dialect.name == "sqlite" + else struct.pack(f"<{index.profile.dimension}f", *item["embedding"]) + ) + if len(vector) != 4 * index.profile.dimension or not all( + math.isfinite(value) for value in struct.unpack(f"<{index.profile.dimension}f", vector) + ): + return False + if index.source == "content" and vector != vector_bytes( + part[index.vector_field], index.profile + ): + return False + if any( + ( + item["scope_id"], + item["family"], + item["revision"], + item["schema_version"], + item["governance_generation"], + item["index_def_id"], + ) + != ( + scope, + family, + head["revision"], + schema, + head["governance_generation"], + definition["index_def_id"], + ) + for item in projected + ): + return False + return True diff --git a/src/powercontext/builtin/persistence/artifact_search_schema.py b/src/powercontext/builtin/persistence/artifact_search_schema.py new file mode 100644 index 0000000000..b60e65568d --- /dev/null +++ b/src/powercontext/builtin/persistence/artifact_search_schema.py @@ -0,0 +1,248 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared search build state and exact, index-isolated projections.""" + +from sqlalchemy import ( + JSON, + BigInteger, + CheckConstraint, + Column, + ForeignKeyConstraint, + Index, + Integer, + LargeBinary, + Table, + Text, + UniqueConstraint, +) + + +def artifact_search_tables(metadata, identity): + spaces = Table( + "pc_artifact_vector_spaces", + metadata, + Column("space_id", identity(32), primary_key=True), + Column("profile_fingerprint", identity(64), nullable=False), + Column("profile_json", JSON, nullable=False), + Column("dimension", Integer, nullable=False), + Column("distance", identity(16), nullable=False), + Column("normalization", identity(16), nullable=False), + Column("storage_format_version", identity(64), nullable=False), + Column("backend", identity(32), nullable=False), + Column("physical_table", identity(128), nullable=False), + Column("provision_state", identity(32), nullable=False), + UniqueConstraint( + "backend", "profile_fingerprint", "storage_format_version", name="uq_pc_artifact_vector_space" + ), + CheckConstraint("dimension > 0", name="ck_pc_artifact_vector_dimension"), + ) + definitions = Table( + "pc_artifact_vector_indexes", + metadata, + Column("index_def_id", identity(64), primary_key=True), + Column("family", identity(128), nullable=False), + Column("index_name", identity(128), nullable=False), + Column("schema_version", identity(128), nullable=False), + Column("projection_version", identity(128), nullable=False), + Column("space_id", identity(32)), + Column("source_kind", identity(16), nullable=False), + Column("projection_spec", JSON, nullable=False), + Column("definition_fingerprint", identity(64), nullable=False), + UniqueConstraint( + "family", + "index_name", + "schema_version", + "projection_version", + "space_id", + "definition_fingerprint", + name="uq_pc_artifact_index_definition", + ), + ForeignKeyConstraint(("space_id",), ("pc_artifact_vector_spaces.space_id",)), + ForeignKeyConstraint( + ("family", "schema_version"), + ("pc_artifact_family_schemas.family", "pc_artifact_family_schemas.schema_version"), + ), + ) + states = Table( + "pc_artifact_search_states", + metadata, + Column("scope_id", identity(256), primary_key=True), + Column("family", identity(128), primary_key=True), + Column("build_id", identity(32)), + Column("state", identity(16), nullable=False), + Column("manifest", JSON, nullable=False), + Column("cleanup_targets", JSON, nullable=False), + Column("validation_token", identity(32)), + Column("start_seq", BigInteger, nullable=False, server_default="0"), + Column("applied_seq", BigInteger, nullable=False, server_default="0"), + Column("change_seq", BigInteger, nullable=False, server_default="0"), + ) + coverage = Table( + "pc_artifact_index_coverage", + metadata, + Column("scope_id", identity(256), primary_key=True), + Column("family", identity(128), primary_key=True), + Column("build_id", identity(32), primary_key=True), + Column("index_def_id", identity(64), primary_key=True), + Column("artifact_id", identity(128), primary_key=True), + Column("revision", Integer, nullable=False), + Column("schema_version", identity(128), nullable=False), + Column("fragment_count", Integer, nullable=False), + Column("projection_input_hash", identity(64), nullable=False), + Column("governance_generation", BigInteger, nullable=False), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + CheckConstraint("fragment_count >= 0", name="ck_pc_artifact_index_fragment_count"), + ) + changes = Table( + "pc_artifact_index_changes", + metadata, + Column("scope_id", identity(256), primary_key=True), + Column("family", identity(128), primary_key=True), + Column("seq", BigInteger, primary_key=True), + Column("artifact_id", identity(128), nullable=False), + Column("revision", Integer), + Column("operation", identity(16), nullable=False), + Column("schema_version", identity(128)), + Column("governance_generation", BigInteger, nullable=False), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + ) + + def rows(name, vector): + return Table( + name, + metadata, + Column("vector_id" if vector else "text_id", identity(64), primary_key=True), + Column("scope_id", identity(256), nullable=False), + Column("family", identity(128), nullable=False), + Column("artifact_id", identity(128), nullable=False), + Column("revision", Integer, nullable=False), + Column("schema_version", identity(128), nullable=False), + Column("index_def_id", identity(64), nullable=False), + Column("build_id", identity(32), nullable=False), + Column("fragment_id", identity(128), nullable=False), + Column("fragment_locator", Text, nullable=False), + Column("input_hash", identity(64), nullable=False), + Column("governance_generation", BigInteger, nullable=False), + *( + ( + Column("space_id", identity(32), nullable=False), + Column("dimension", Integer, nullable=False), + Column("embedding", LargeBinary, nullable=False), + ) + if vector + else (Column("searchable_text", Text, nullable=False),) + ), + UniqueConstraint("build_id", "index_def_id", "artifact_id", "fragment_id", name=f"uq_{name}_fragment"), + ForeignKeyConstraint( + ("scope_id", "family", "artifact_id", "revision"), + ("pc_artifacts.scope_id", "pc_artifacts.family", "pc_artifacts.artifact_id", "pc_artifacts.revision"), + ), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + Index( + f"ix_{name}_search", + "scope_id", + "family", + "build_id", + *(("space_id",) if vector else ()), + "index_def_id", + "artifact_id", + ), + *( + ( + CheckConstraint( + "dimension > 0 AND length(embedding) = 4 * dimension", name="ck_pc_artifact_vectors_encoding" + ), + ) + if vector + else () + ), + ) + + return ( + spaces, + definitions, + states, + coverage, + changes, + rows("pc_artifact_vectors", True), + rows("pc_artifact_search_text", False), + ) + + +def ensure_search_support(connection): # noqa: C901 - Validate the fixed layout before repairing derived FTS support. + """Create missing backend full-text support during shared database initialization.""" + from sqlalchemy import inspect, text + + from powercontext.builtin.persistence.tables import ARTIFACT_SEARCH_TABLES + + inspector = inspect(connection) + for table in ARTIFACT_SEARCH_TABLES: + if table.name == "pc_artifact_vectors" and connection.dialect.name != "sqlite": + continue + name, expected = table.name, set(table.c.keys()) + if {column["name"] for column in inspector.get_columns(name)} != expected: + raise ValueError("Artifact search schema is incompatible; a database migration is required") # noqa: TRY003 + if inspector.has_table("pc_artifact_index_generations"): + raise ValueError("Legacy Artifact search generations require an explicit database migration") # noqa: TRY003 + if connection.dialect.name == "sqlite" and inspector.has_table("pc_artifact_search_fts"): + columns = {column["name"] for column in inspector.get_columns("pc_artifact_search_fts")} + if not {"build_id", "index_def_id"}.issubset(columns): + raise ValueError("Artifact full-text schema is incompatible; a database migration is required") # noqa: TRY003 + if connection.dialect.name != "sqlite": + indexes = {index["name"] for index in inspect(connection).get_indexes("pc_artifact_search_text")} + if "ix_pc_artifact_search_fts" not in indexes: + connection.exec_driver_sql( + "CREATE FULLTEXT INDEX ix_pc_artifact_search_fts " + "ON pc_artifact_search_text (searchable_text) WITH PARSER SPACE" + ) + return + helpers = ( + ( + "pc_artifact_search_fts", + "CREATE VIRTUAL TABLE pc_artifact_search_fts USING fts5(text_id UNINDEXED, scope_id UNINDEXED, family UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED, build_id UNINDEXED, index_def_id UNINDEXED, searchable_text, tokenize='unicode61')", + ), + ( + "pc_artifact_search_fts_insert", + "CREATE TRIGGER pc_artifact_search_fts_insert AFTER insert ON pc_artifact_search_text BEGIN INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END", + ), + ( + "pc_artifact_search_fts_update", + "CREATE TRIGGER pc_artifact_search_fts_update AFTER update ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id;INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END", + ), + ( + "pc_artifact_search_fts_delete", + "CREATE TRIGGER pc_artifact_search_fts_delete AFTER delete ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id; END", + ), + ) + missing = False + for name, ddl in helpers: + if connection.scalar(text("SELECT 1 FROM sqlite_master WHERE name=:name"), {"name": name}) is None: + connection.exec_driver_sql(ddl) + missing = True + if missing: + # Repair derived FTS rows only when support was missing, never on normal startup. + connection.exec_driver_sql("DELETE FROM pc_artifact_search_fts") + connection.exec_driver_sql( + "INSERT INTO pc_artifact_search_fts " + "(text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) " + "SELECT text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text " + "FROM pc_artifact_search_text" + ) diff --git a/src/powercontext/builtin/persistence/artifact_vector_space.py b/src/powercontext/builtin/persistence/artifact_vector_space.py new file mode 100644 index 0000000000..0ed113723f --- /dev/null +++ b/src/powercontext/builtin/persistence/artifact_vector_space.py @@ -0,0 +1,243 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Server-owned OceanBase/seekDB VECTOR spaces, prepared from durable declarations.""" + +import re +from hashlib import sha256 + +from pyobvector import VECTOR +from sqlalchemy import ( + Column, + ForeignKeyConstraint, + Index, + MetaData, + Table, + UniqueConstraint, + select, + update, +) + +from powercontext.builtin.artifacts.memory.canonical import canonical_json +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_VECTOR_INDEXES_TABLE, + ARTIFACT_VECTOR_SPACES_TABLE, + ARTIFACT_VECTORS_TABLE, + SHARED_METADATA, +) + + +def native_vector_table(space): + identity = space["space_id"] + if re.fullmatch(r"[0-9a-f]{32}", identity) is None or space["physical_table"] != f"pc_av_{identity}": + raise ValueError("invalid registered vector space table identity") # noqa: TRY003 + profile = EmbeddingProfile.model_validate(space["profile_json"]) + fingerprint = sha256(canonical_json(profile.model_dump(mode="json"))).hexdigest() + if ( + (profile.dimension, profile.distance, profile.normalization, fingerprint) + != (space["dimension"], space["distance"], space["normalization"], space["profile_fingerprint"]) + or profile.dimension < 1 + or space["storage_format_version"] != "float32-le-v1" + ): + raise ValueError("inconsistent vector space profile or storage format") # noqa: TRY003 + metadata = MetaData() + # Foreign key targets resolve in isolated metadata without provisioning them here. + for name in ("pc_artifacts", "pc_artifact_search_states"): + target = SHARED_METADATA.tables[name] + Table( + name, + metadata, + *(Column(column.name, column.type, primary_key=True) for column in target.primary_key.columns), + ) + return Table( + space["physical_table"], + metadata, + *( + Column( + column.name, + VECTOR(profile.dimension) if column.name == "embedding" else column.type, + primary_key=column.primary_key, + nullable=column.nullable, + ) + for column in ARTIFACT_VECTORS_TABLE.columns + ), + UniqueConstraint("build_id", "index_def_id", "artifact_id", "fragment_id", name=f"uq_av_{identity}"), + ForeignKeyConstraint( + ("scope_id", "family", "artifact_id", "revision"), + ("pc_artifacts.scope_id", "pc_artifacts.family", "pc_artifacts.artifact_id", "pc_artifacts.revision"), + ), + ForeignKeyConstraint( + ("scope_id", "family"), ("pc_artifact_search_states.scope_id", "pc_artifact_search_states.family") + ), + Index(f"ix_av_{identity}", "scope_id", "family", "build_id", "space_id", "index_def_id", "artifact_id"), + ) + + +def native_space_layout(space): + """The one exact framework template shared by preparation and migration.""" + from sqlalchemy.dialects import mysql + from sqlalchemy.schema import CreateTable + + from powercontext.builtin.persistence.migrations.mysql_schema import _parse_table + + table = native_vector_table(space) + _, layout = _parse_table(str(CreateTable(table).compile(dialect=mysql.dialect()))) + for index in table.indexes: + layout["indexes"][index.name] = { + "columns": [[column.name, None, "asc"] for column in index.columns], + "unique": index.unique, + } + return layout + + +def inspect_native_space(space, actual, *, allow_incomplete=False): + """Return missing indexes only for an exact, declared creation intermediate.""" + import copy + + from powercontext.builtin.persistence.migrations.mysql_schema import _matches + + expected = native_space_layout(space) + if space["backend"] != "mysql" or space["provision_state"] not in {"declared", "ready"}: + raise ValueError("invalid native vector space declaration") # noqa: TRY003 + incomplete = allow_incomplete and space["provision_state"] == "declared" + if actual is None: + if incomplete: + return tuple(expected["indexes"]) + raise ValueError("vector space physical table is absent") # noqa: TRY003 + partial = copy.deepcopy(expected) + missing = tuple(name for name in expected["indexes"] if name not in actual["indexes"]) + # Unique constraints are part of CREATE TABLE, only standalone indexes can + # be absent after interruption between table and index creation. + allowed_missing = {index.name for index in native_vector_table(space).indexes} + if missing and (not incomplete or not set(missing) <= allowed_missing): + raise ValueError("vector space required indexes are absent") # noqa: TRY003 + for name in missing: + del partial["indexes"][name] + if not _matches({"tables": {"space": actual}}, {"tables": {"space": partial}}): + raise ValueError("vector space physical structure differs from its registered format") # noqa: TRY003 + return missing + + +def registered_native_layouts(declarations, tables, *, allow_declared=False): + """Admit only declared identities with the exact supported physical format.""" + layouts = {} + names = set() + for space in declarations: + name = space["physical_table"] + if name in names or (space["provision_state"] != "ready" and not allow_declared): + raise ValueError("native vector space preparation is incomplete or duplicated") # noqa: TRY003 + names.add(name) + actual = tables.get(name) + inspect_native_space(space, actual, allow_incomplete=allow_declared) + if actual is not None: + layouts[name] = actual + return layouts + + +def _physical_layout(connection, name): + from sqlalchemy import inspect + + from powercontext.builtin.persistence.migrations.mysql_schema import _parse_table + + if not inspect(connection).has_table(name): + return None + # The name has already passed native_vector_table's fixed hex identity check. + row = connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`").one() + database = connection.exec_driver_sql("SELECT DATABASE()").scalar_one() + return _parse_table(row[1], database=database)[1] + + +def verify_native_space(connection, space): + table = native_vector_table(space) + inspect_native_space(space, _physical_layout(connection, table.name)) + + +def provision_native_space(connection, space): + """Converge using atomic database DDL, then inspect before publishing ready.""" + from sqlalchemy.exc import DBAPIError + from sqlalchemy.schema import CreateTable + + table = native_vector_table(space) + actual = _physical_layout(connection, table.name) + missing = inspect_native_space(space, actual, allow_incomplete=True) + if space["provision_state"] == "ready": + return + if actual is None: + # IF NOT EXISTS serializes competing CREATEs at the backend. It is not + # admission: the resulting definition must still match exactly. + connection.execute(CreateTable(table, if_not_exists=True)) + missing = inspect_native_space(space, _physical_layout(connection, table.name), allow_incomplete=True) + for index in table.indexes: + if index.name not in missing: + continue + try: + index.create(connection) + except DBAPIError as error: + # A concurrent creator may have won this particular index. No other + # failure is normalized, and the full structure is checked below. + arguments = getattr(error.orig, "args", ()) + if not arguments or arguments[0] != 1061: + raise + verify_native_space(connection, space) + connection.execute( + update(ARTIFACT_VECTOR_SPACES_TABLE) + .where( + ARTIFACT_VECTOR_SPACES_TABLE.c.space_id == space["space_id"], + ARTIFACT_VECTOR_SPACES_TABLE.c.provision_state == "declared", + ) + .values(provision_state="ready") + ) + + +async def prepare_family_spaces(database, family): + """Await only this family's durable spaces, outside registration transactions.""" + if database.engine.dialect.name == "sqlite": + return + spaces = ARTIFACT_VECTOR_SPACES_TABLE + definitions = ARTIFACT_VECTOR_INDEXES_TABLE + async with database.transaction() as connection: + declarations = ( + ( + await connection.execute( + select(spaces).where( + spaces.c.space_id.in_(select(definitions.c.space_id).where(definitions.c.family == family)) + ) + ) + ) + .mappings() + .all() + ) + for space in declarations: + # DDL may commit independently. This connection owns no content or + # plugin registration writes; ready is committed only after verification. + async with database.transaction() as connection: + current = ( + (await connection.execute(select(spaces).where(spaces.c.space_id == space["space_id"]))) + .mappings() + .one() + ) + await connection.run_sync(lambda sync, current=current: provision_native_space(sync, current)) + + +def provision_native_spaces(connection): + """Prepare persisted native declarations during unmanaged initialization.""" + spaces = ( + connection + .execute(select(ARTIFACT_VECTOR_SPACES_TABLE).where(ARTIFACT_VECTOR_SPACES_TABLE.c.backend == "mysql")) + .mappings() + .all() + ) + for space in spaces: + provision_native_space(connection, space) diff --git a/src/powercontext/builtin/persistence/custom_artifacts.py b/src/powercontext/builtin/persistence/custom_artifacts.py new file mode 100644 index 0000000000..976c8819f0 --- /dev/null +++ b/src/powercontext/builtin/persistence/custom_artifacts.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Durable declarations and JSON storage for user-defined Artifact families.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, JsonValue +from sqlalchemy import inspect +from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.schema import CreateColumn +from typing_extensions import override + +from powercontext.artifacts import Artifact +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.tables import ARTIFACT_PROCESSING_BINDING_STATES_TABLE + + +class StoredArtifactContent(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: str + content: dict[str, JsonValue] + + +def custom_artifact_repository( + family: str, sources: SourceRepository | None = None, *, prepared=None +) -> ArtifactRepository: + artifact_type = type("CustomArtifact", (Artifact[StoredArtifactContent],), {"family": family}) + return _CustomArtifactRepository((artifact_type,), sources=sources, prepared=prepared) + + +class _CustomArtifactRepository(ArtifactRepository): + def __init__(self, artifact_types, *, sources=None, prepared=None): + super().__init__(artifact_types, sources=sources) + self.prepared = prepared or {} + + @override + async def create(self, connection, scope_id, artifact_id, draft, /, **kwargs): + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher + + publisher = ArtifactProjectionPublisher(None) + prepared = self.prepared.get(artifact_id) + await publisher.validate_prepared( + connection, scope_id, draft.family, draft.content.schema_version, draft.content.content, prepared + ) + result = await super().create(connection, scope_id, artifact_id, draft, **kwargs) + await publisher.publish(connection, scope_id, result, prepared) + return result + + @override + async def lock_heads(self, connection, scope_id, refs, /): + from powercontext.builtin.persistence.artifact_indexes import lock_state + + for family in sorted({ref.family for ref in refs}): + await lock_state(connection, scope_id, family) + return await super().lock_heads(connection, scope_id, refs) + + @override + async def _revise(self, connection, scope_id, artifact, draft, **kwargs): + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher + + publisher = ArtifactProjectionPublisher(None) + prepared = self.prepared.get(artifact.artifact_id) + await publisher.validate_prepared( + connection, scope_id, draft.family, draft.content.schema_version, draft.content.content, prepared + ) + result = await super()._revise(connection, scope_id, artifact, draft, **kwargs) + await publisher.publish(connection, scope_id, result, prepared) + return result + + +async def ensure_artifact_plugin_schema(connection: AsyncConnection) -> None: + """Add registration fields to existing databases without replacing processing state.""" + + existing = await connection.run_sync( + lambda sync: {item["name"] for item in inspect(sync).get_columns(ARTIFACT_PROCESSING_BINDING_STATES_TABLE.name)} + ) + for column in ARTIFACT_PROCESSING_BINDING_STATES_TABLE.columns: + if column.name not in existing: + definition = str(CreateColumn(column).compile(dialect=connection.dialect)) + await connection.exec_driver_sql( + f"ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN {definition}" + ) diff --git a/src/powercontext/builtin/persistence/lexical_projection.py b/src/powercontext/builtin/persistence/lexical_projection.py index b1f564e3ed..c67aa3c902 100644 --- a/src/powercontext/builtin/persistence/lexical_projection.py +++ b/src/powercontext/builtin/persistence/lexical_projection.py @@ -19,7 +19,7 @@ from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncConnection -from powercontext.builtin.artifacts.search import analyze_text +from powercontext.builtin.artifacts.search import TEXT_ANALYZER_VERSION, analyze_text from powercontext.builtin.persistence.tables import ( LEXICAL_PROJECTION_VERSIONS_TABLE, TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, @@ -27,7 +27,7 @@ ) # Bump when document segmentation or token encoding changes; query stopwords do not affect projections. -LEXICAL_ANALYZER_VERSION = "jieba-0.42.1-words-v1" +LEXICAL_ANALYZER_VERSION = TEXT_ANALYZER_VERSION async def refresh_lexical_projections(connection: AsyncConnection, atomic_table: Table | None) -> None: diff --git a/src/powercontext/builtin/persistence/migrations/maintenance.py b/src/powercontext/builtin/persistence/migrations/maintenance.py index eb44bca878..e55a7e01b9 100644 --- a/src/powercontext/builtin/persistence/migrations/maintenance.py +++ b/src/powercontext/builtin/persistence/migrations/maintenance.py @@ -207,12 +207,22 @@ def execute(connection, identity, guard): backup_state="pending", ) journal.write(evidence) + backup_objects = self.bundle.affected_objects + if identity.product != "sqlite": + from .mysql_schema import mysql_inventory + + backup_objects = tuple( + dict.fromkeys(( + *backup_objects, + *(space["physical_table"] for space in mysql_inventory(connection).get("artifact_spaces", [])), + )) + ) context = BackupContext( database_id=identity.database_id, source_revision=self.bundle.head, bundle_checksum=self.checksum, maintenance_window_id=evidence.maintenance_window_id, - objects=self.bundle.affected_objects, + objects=backup_objects, writers_stopped=True, ) provider = self._provider(connection, identity) diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index ea7706d431..a0759983fb 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -267,6 +267,7 @@ def plan( backup_policy: BackupPolicy = "auto", service: dict[str, Any] | None = None, shared_database: bool = False, + _startup: bool = False, require_existing_database: bool = False, ) -> MigrationPlan: self._assert_unchanged() @@ -275,7 +276,12 @@ def inspect( connection: Connection | None, identity: BackendIdentity, _guard: Callable[[], None] ) -> MigrationPlan: return self._inspect( - connection, identity, backup_policy=backup_policy, service=service, shared_database=shared_database + connection, + identity, + backup_policy=backup_policy, + service=service, + shared_database=shared_database, + allow_declared_spaces=_startup, ) if isinstance(self.connections.config, SeekDBConfig) and not require_existing_database: @@ -540,6 +546,7 @@ def _inspect( # noqa: C901 -- Keep ordered admission checks with their operatio backup_policy: BackupPolicy, service: dict[str, Any] | None = None, shared_database: bool = False, + allow_declared_spaces: bool = False, ) -> MigrationPlan: if backup_policy not in {"auto", "manual", "skip"}: raise MigrationError("invalid_backup_policy", "Choose auto, manual or skip.") @@ -588,6 +595,7 @@ def _inspect( # noqa: C901 -- Keep ordered admission checks with their operatio "server_migration_unavailable", f"The complete {source.release} Server schema is recognized, but its full upgrade is unavailable.", ) + self.schema.allow_declared_spaces = allow_declared_spaces and current == self.bundle.head current, adopt = self._recognize_source(inventory, initialized, current, evidence) if initialized and self.bundle.pending(current) and evidence is None: raise MigrationError("recovery_required", "The original external maintenance evidence is missing.") @@ -646,7 +654,7 @@ def _inspect( # noqa: C901 -- Keep ordered admission checks with their operatio "backup_policy": backup_policy, "backup_available": available, "backup_reason": reasons, - "affected_objects": self.bundle.affected_objects, + "affected_objects": self.schema.affected_objects, "retained_objects": self.bundle.retained_objects, "service": service, "shared_database": shared_database, diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 4364a4fde9..1896a2b876 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -93,6 +93,19 @@ def mysql_inventory(connection: Connection, *, exclude: frozenset[str] = frozens key=lambda row: str(row.get("Trigger", "")), ) inventory["views"].sort() + if "pc_artifact_vector_spaces" in inventory["tables"]: + from sqlalchemy import select + + from powercontext.builtin.persistence.tables import ARTIFACT_VECTOR_SPACES_TABLE + + declarations = ( + connection + .execute(select(ARTIFACT_VECTOR_SPACES_TABLE).order_by(ARTIFACT_VECTOR_SPACES_TABLE.c.space_id)) + .mappings() + .all() + ) + if declarations: + inventory["artifact_spaces"] = [dict(space) for space in declarations] except DBAPIError as error: raise MigrationError( "schema_reflection_failed", "Complete database schema reflection is unavailable." @@ -141,7 +154,10 @@ def read_revision( class MySQLSchema: """Validate source, intermediate and post-DDL shapes from frozen resources.""" - def __init__(self, bundle: MigrationBundle, *, embedded_code: bool = False) -> None: # noqa: C901 -- Keep ordered admission checks with their operation. + def __init__( # noqa: C901 -- Keep ordered admission checks with their operation. + self, bundle: MigrationBundle, *, embedded_code: bool = False, allow_declared_spaces: bool = False + ) -> None: + self.allow_declared_spaces = allow_declared_spaces self.bundle = bundle self.revisions = bundle.revisions manifest = json.loads((bundle.snapshot_directory / "manifest.json").read_text()) @@ -338,13 +354,27 @@ def _extend_vectors(self) -> None: self.affected_objects = tuple(dict.fromkeys([*self.affected_objects, *tables, declaration_name])) def _matches(self, inventory: dict[str, Any], expected: dict[str, Any]) -> bool: - # Most DDL recovery prefixes have different table sets. Reject them - # before normalizing backend defaults for every table. - if inventory.keys() != expected.keys() or inventory["tables"].keys() != expected["tables"].keys(): - return False - # Captured backends may expose an explicit default that another backend - # omits. Only registered values may be absent; retain it in fingerprints. actual = {**inventory, "tables": dict(inventory["tables"])} + declarations = actual.pop("artifact_spaces", []) + if declarations: + from powercontext.builtin.persistence.artifact_vector_space import registered_native_layouts + + if "pc_artifact_vector_spaces" not in expected["tables"]: + return False + try: + layouts = registered_native_layouts( + declarations, actual["tables"], allow_declared=self.allow_declared_spaces + ) + except (ValueError, TypeError, KeyError) as error: + raise MigrationError( + "incompatible_schema", "A registered native vector space is incomplete or incompatible." + ) from error + expected = {**expected, "tables": dict(expected["tables"])} + expected["tables"].update(layouts) + self.affected_objects = tuple(dict.fromkeys((*self.affected_objects, *sorted(layouts)))) + # Compare table sets after adding the exactly validated registered spaces. + if actual.keys() != expected.keys() or actual["tables"].keys() != expected["tables"].keys(): + return False for name, options in self.optional_table_options.items(): table = actual["tables"].get(name) target = expected["tables"].get(name) diff --git a/src/powercontext/builtin/persistence/migrations/server_readiness.py b/src/powercontext/builtin/persistence/migrations/server_readiness.py index 6771f46845..a27469a876 100644 --- a/src/powercontext/builtin/persistence/migrations/server_readiness.py +++ b/src/powercontext/builtin/persistence/migrations/server_readiness.py @@ -24,7 +24,7 @@ def check_server_schema(connection, *, _archive_recovery=False, _startup=False): - bundle = check_server_layout(connection) + bundle = check_server_layout(connection, allow_declared_spaces=_startup) if _startup and connection.dialect.name == "sqlite": _check_sqlite_startup_history(connection, bundle) pending_restore = connection.exec_driver_sql( @@ -47,7 +47,7 @@ def check_server_schema(connection, *, _archive_recovery=False, _startup=False): bundle.domains.validate(connection, bundle.head, backend=backend, configuration=configuration) -def check_server_layout(connection): +def check_server_layout(connection, *, allow_declared_spaces=False): """Validate the exact managed layout before a registered domain repair.""" from .deployment import server_bundle from .mysql_schema import MySQLSchema, mysql_inventory @@ -76,7 +76,7 @@ def check_server_layout(connection): embedded_code = connection.engine.url.drivername == "mysql+aseekdb" and any( name.startswith("pc_code_") for name in inventory["tables"] ) - schema = MySQLSchema(bundle, embedded_code=embedded_code) + schema = MySQLSchema(bundle, embedded_code=embedded_code, allow_declared_spaces=allow_declared_spaces) schema.verify(inventory, bundle.head) return bundle @@ -135,16 +135,18 @@ def verify(connection, identity, guard): raise MigrationError( "migration_required", "Initialize the remote database with an explicit db-migrate job." ) - bundle = check_server_layout(connection) + bundle = check_server_layout(connection, allow_declared_spaces=True) stored = bundle.vector_profile.profile if bundle.vector_profile is not None else None if stored != profile: raise MigrationError("embedding_profile_mismatch", "Server configuration differs from managed storage.") - check_server_schema(connection) + check_server_schema(connection, _startup=True) MaintenanceConnections(database).run(verify) return runner = deployment_runner(settings, embedding_profile=profile) - plan = runner.plan() + from .mysql import MySQLMigrationRunner + + plan = runner.plan(_startup=True) if isinstance(runner, MySQLMigrationRunner) else runner.plan() if plan.state == "uninitialized" and database.kind in {"sqlite", "seekdb"}: runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) elif plan.state != "ready": diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/artifact_storage_readiness.py b/src/powercontext/builtin/persistence/migrations/server_resources/artifact_storage_readiness.py new file mode 100644 index 0000000000..bc7b94c4ac --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/artifact_storage_readiness.py @@ -0,0 +1,161 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Server readiness with durable Artifact plugin bindings at v1.5.0.001. + +The live validation retains the previous Server checks. Its only new admission +rule recognizes persisted plugin bindings, including unloaded registrations. +No business data or projection transformation accompanies this revision. +""" + +import hashlib +import json +import runpy +from pathlib import Path + +from sqlalchemy import text + +from powercontext.builtin.persistence.experience_index import APPROVED_ARTIFACT_PROJECTION_VERSION +from powercontext.builtin.persistence.lexical_projection import LEXICAL_ANALYZER_VERSION +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.pinned import finish, pinned + +core = runpy.run_path(str(Path(__file__).parent / "server_domain.py")) +vectors = runpy.run_path(str(Path(__file__).parent / "server_vector_domain.py")) + + +def preflight(connection, *, configuration): + """The prior revision owns source admission before fixed schema expansion.""" + + +def apply(connection, *, configuration): + """Plugin storage requires no rewrite of existing records or projections.""" + + +def verify(connection, *, configuration): + validate(connection, configuration=configuration) + + +def validate(connection, *, configuration): # noqa: C901 -- Preserve the complete versioned Server readiness contract. + vector = configuration.get("embedding_profile") is not None + if vector: + vectors["_validate_vector_configuration"](connection, configuration) + vectors["_require_declaration"](connection, configuration) + finish(core["assert_atomic_memory_migration_ready"](pinned(connection))) + expected = { + "pc_atomic_memory_current": LEXICAL_ANALYZER_VERSION, + "pc_topic_memory_active_topics": LEXICAL_ANALYZER_VERSION, + "pc_topic_memory_active_chunks": LEXICAL_ANALYZER_VERSION, + "pc_artifact_heads": APPROVED_ARTIFACT_PROJECTION_VERSION, + } + actual = dict( + connection + .exec_driver_sql("SELECT projection, analyzer_version FROM pc_lexical_projection_versions") + .tuples() + .all() + ) + if actual != expected: + raise MigrationError("projection_migration_required", "The registered search projections are not current.") + frozen = core["dream"]() + manifest = frozen["_processing_manifest"](connection, frozen["_schema"](), upgraded=True) + _validate_intent_bindings(connection) + target = configuration.get("processing_manifest") + if target and any(manifest[key] != target[key] for key in ("bindings", "mode", "capabilities")): + raise MigrationError( + "incompatible_processing_configuration", "Processing ownership differs from the migrated declaration." + ) + from powercontext.builtin.persistence.dream import DreamRepository + + for row in connection.exec_driver_sql( + "SELECT scope_id,run_id,status,operation,generation,request_generation,principal_key,idempotency_key FROM pc_dream_runs" + ).mappings(): + record = finish(DreamRepository().get(pinned(connection), row["scope_id"], row["run_id"])) + if ( + any(getattr(record.run, key) != row[key] for key in ("scope_id", "run_id", "status", "operation")) + or record.generation != row["generation"] + or record.request_generation != row["request_generation"] + or hashlib.sha256(record.principal_id.encode()).hexdigest() != row["principal_key"] + or (record.request is not None and record.request.idempotency_key != row["idempotency_key"]) + ): + raise MigrationError("unsupported_task_format", "A Dream payload disagrees with its durable task identity.") + from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository + + for scope, binding in connection.exec_driver_sql( + "SELECT scope_id,binding_name FROM pc_artifact_processing_intents" + ): + finish(ArtifactProcessingIntentRepository().load(pinned(connection), scope, binding)) + from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + from powercontext.builtin.runtime.atomic_memory_rebuild import verify_atomic_memory_current + + profile = vectors["_profile"](configuration) if vector else None + atomic_index = ( + SQLiteAtomicMemoryIndex(profile) if connection.dialect.name == "sqlite" else OceanBaseAtomicMemoryIndex(profile) + ) + report = finish(verify_atomic_memory_current(pinned(connection), atomic_index)) + if not report.ready: + raise MigrationError( + "projection_verification_failed", "Atomic Memory projections differ from their authoritative records." + ) + + from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository + + if connection.dialect.name == "sqlite": + from powercontext.builtin.persistence.sqlite.topic_memory_index import SQLiteTopicMemoryFTSIndex + + index = SQLiteTopicMemoryFTSIndex() + core["_validate_sqlite_fts"](connection) + if vector: + vectors["_validate_atomic_helper"](connection, configuration) + else: + from powercontext.builtin.persistence.oceanbase.topic_memory_index import OceanBaseTopicMemoryFTSIndex + + index = OceanBaseTopicMemoryFTSIndex() + if vector: + index = vectors["_topic_index"](connection, configuration) + finish(TopicMemoryRepository(index=index).initialize(pinned(connection), validate_only=True)) + + +def _validate_intent_bindings(connection): + manifest = json.loads( + connection.exec_driver_sql( + "SELECT config_manifest FROM pc_artifact_processing_schema WHERE singleton=1" + ).scalar_one() + ) + for (binding,) in connection.exec_driver_sql("SELECT DISTINCT binding_name FROM pc_artifact_processing_intents"): + if binding in manifest["bindings"]: + continue + registration = connection.execute( + text( + "SELECT b.plugin_id,b.plugin_family,b.plugin_spec FROM pc_artifact_processing_binding_states AS b " + "WHERE b.binding_name=:binding AND b.plugin_id IS NOT NULL AND b.plugin_family IS NOT NULL " + "AND EXISTS (SELECT 1 FROM pc_artifact_family_schemas AS s WHERE s.family=b.plugin_family)" + ), + {"binding": binding}, + ).first() + if registration is None: + raise MigrationError("unsupported_task_format", "A pending Processing intent has no registered binding.") + plugin_id, family, specification = registration + # Unload retains the durable binding identity and family schemas while + # removing deployable code configuration. Its pending evidence stays valid. + if specification is None: + continue + specification = json.loads(specification) if isinstance(specification, str) else specification + if ( + specification["manifest"]["binding_name"] != binding + or specification["manifest"]["id"] != plugin_id + or specification["declaration"]["plugin_id"] != plugin_id + or specification["declaration"]["family"] != family + ): + raise MigrationError("unsupported_task_format", "A plugin binding disagrees with its durable declaration.") diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json index deece2d1b8..c3b8d5de64 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json @@ -29,6 +29,10 @@ "schemas/sqlite-v1_4_1_005.json", "schemas/mysql-v1_4_1_005.json" ], + "v1_5_0_001": [ + "schemas/sqlite-v1_5_0_001.json", + "schemas/mysql-v1_5_0_001.json" + ], "v1_4_1_006": [ "schemas/sqlite-v1_4_1_006.json", "schemas/mysql-v1_4_1_006.json" @@ -56,6 +60,9 @@ "v1_4_1_005": [ "2e24be9f699c202cb9d7985690b2be947bdc98f8f64c1c4e4b8e16e35a580fc4" ], + "v1_5_0_001": [ + "8f2d4a32f3558612b1bd390c262b8b3e355b172e34cc99692f38c11876226ebd" + ], "v1_4_1_006": [ "cacce80bdfb14aff889b7babebc9ab8487b7f0a11d2dbbd3d77a3717a23df74c" ] @@ -63,12 +70,14 @@ "affected_objects": [ "pc_access_audit", "pc_access_idempotency", + "pc_access_owner_recovery", "pc_access_owners", "pc_access_relationship_heads", "pc_access_relationships", "pc_agent_skill_targets", "pc_artifact_candidate_heads", "pc_artifact_candidate_versions", + "pc_artifact_family_schemas", "pc_artifact_fts", "pc_artifact_fts_config", "pc_artifact_fts_content", @@ -77,6 +86,8 @@ "pc_artifact_fts_idx", "pc_artifact_handles", "pc_artifact_heads", + "pc_artifact_index_changes", + "pc_artifact_index_coverage", "pc_artifact_lineage_artifacts", "pc_artifact_lineage_sources", "pc_artifact_processing_auto_wave_targets", @@ -88,7 +99,21 @@ "pc_artifact_processing_schema", "pc_artifact_processing_sequences", "pc_artifact_publications", + "pc_artifact_search_fts", + "pc_artifact_search_fts_config", + "pc_artifact_search_fts_content", + "pc_artifact_search_fts_data", + "pc_artifact_search_fts_delete", + "pc_artifact_search_fts_docsize", + "pc_artifact_search_fts_idx", + "pc_artifact_search_fts_insert", + "pc_artifact_search_fts_update", + "pc_artifact_search_states", + "pc_artifact_search_text", "pc_artifact_tags", + "pc_artifact_vector_indexes", + "pc_artifact_vector_spaces", + "pc_artifact_vectors", "pc_artifacts", "pc_atomic_memory_current", "pc_atomic_memory_current_fts", @@ -105,16 +130,20 @@ "pc_dream_runs", "pc_external_skill_registrations", "pc_lexical_projection_versions", + "pc_memory_artifact_archive", "pc_memory_entry_heads", "pc_memory_entry_versions", "pc_memory_source_windows", "pc_model_usage_daily", + "pc_portable_restore_receipts", "pc_profile_policies", "pc_recall_effort_daily", "pc_recall_token_daily", "pc_receipt_migration_review", "pc_recurrence_match", "pc_recurrence_observation", + "pc_retained_dream_candidate_heads", + "pc_retained_dream_candidate_versions", "pc_scope_bindings", "pc_scope_context_references", "pc_scope_creation_requests", @@ -148,11 +177,6 @@ "pc_topic_memory_topic_fts_idx", "pc_topic_memory_work_blocks", "pc_topic_memory_work_budgets", - "pc_retained_dream_candidate_heads", - "pc_retained_dream_candidate_versions", - "pc_memory_artifact_archive", - "pc_portable_restore_receipts", - "pc_access_owner_recovery", "pc_source_write_plugins" ], "readiness_scope": "server", @@ -194,6 +218,27 @@ "seekdb", "oceanbase" ] + }, + { + "id": "artifact_storage", + "revision": "v1_5_0_001", + "kind": "data", + "module": "artifact_storage_readiness.py", + "resources": [ + "server_domain.py", + "server_vector_domain.py", + "atomic_memory_readiness_1300.py", + "dream/upgrade.py", + "dream/schema.json" + ], + "depends_on": [ + "portable_storage" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] } ], "domain_readiness": { @@ -215,6 +260,10 @@ "server_upgrade", "portable_storage" ], + "v1_5_0_001": [ + "artifact_storage", + "portable_storage" + ], "v1_4_1_006": [ "server_upgrade", "portable_storage" @@ -235,6 +284,7 @@ "schemas/mysql-s1402.json", "schemas/mysql-v1_4_1_004.json", "schemas/mysql-v1_4_1_005.json", + "schemas/mysql-v1_5_0_001.json", "schemas/mysql-v1_4_1_006.json" ], "mysql_history": "mysql", diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_001.py b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_001.py new file mode 100644 index 0000000000..10e33ea9e4 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_001.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Apply registered fixed Artifact storage through the shared DDL executor.""" + +from alembic import context + +revision = "v1_5_0_001" +down_revision = "v1_4_1_006" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_001.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_001.json new file mode 100644 index 0000000000..122c3b914d --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_001.json @@ -0,0 +1,1914 @@ +{ + "format": "powercontext.mysql-state-layouts.v1", + "revisions": { + "v1_5_0_001": { + "statements": [ + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_id VARCHAR(128), ADD COLUMN plugin_family VARCHAR(128), ADD COLUMN plugin_spec JSON, ADD COLUMN binding_generation BIGINT NOT NULL DEFAULT '0', ADD COLUMN desired_state VARCHAR(16), ADD COLUMN phase VARCHAR(16), ADD COLUMN last_error VARCHAR(1024)", + "CREATE TABLE pc_artifact_family_schemas (\n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tschema_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdeclaration JSON NOT NULL, \n\tPRIMARY KEY (family, schema_version)\n)", + "CREATE TABLE pc_artifact_vector_spaces (\n\tspace_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprofile_fingerprint VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprofile_json JSON NOT NULL, \n\tdimension INTEGER NOT NULL, \n\tdistance VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tnormalization VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tstorage_format_version VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tbackend VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tphysical_table VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprovision_state VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (space_id), \n\tCONSTRAINT uq_pc_artifact_vector_space UNIQUE (backend, profile_fingerprint, storage_format_version), \n\tCONSTRAINT ck_pc_artifact_vector_dimension CHECK (dimension > 0)\n)", + "CREATE TABLE pc_artifact_vector_indexes (\n\tindex_def_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tindex_name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tschema_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprojection_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tspace_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tsource_kind VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tprojection_spec JSON NOT NULL, \n\tdefinition_fingerprint VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (index_def_id), \n\tCONSTRAINT uq_pc_artifact_index_definition UNIQUE (family, index_name, schema_version, projection_version, space_id, definition_fingerprint), \n\tFOREIGN KEY(space_id) REFERENCES pc_artifact_vector_spaces (space_id), \n\tFOREIGN KEY(family, schema_version) REFERENCES pc_artifact_family_schemas (family, schema_version)\n)", + "CREATE TABLE pc_artifact_search_states (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tbuild_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tstate VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmanifest JSON NOT NULL, \n\tcleanup_targets JSON NOT NULL, \n\tvalidation_token VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tstart_seq BIGINT NOT NULL DEFAULT '0', \n\tapplied_seq BIGINT NOT NULL DEFAULT '0', \n\tchange_seq BIGINT NOT NULL DEFAULT '0', \n\tPRIMARY KEY (scope_id, family)\n)", + "CREATE TABLE pc_artifact_index_coverage (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tbuild_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tindex_def_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfragment_count INTEGER NOT NULL, \n\tprojection_input_hash VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, family, build_id, index_def_id, artifact_id), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family), \n\tCONSTRAINT ck_pc_artifact_index_fragment_count CHECK (fragment_count >= 0)\n)", + "CREATE TABLE pc_artifact_index_changes (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tseq BIGINT NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER, \n\toperation VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tschema_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tgovernance_generation BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, family, seq), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family)\n)", + "CREATE TABLE pc_artifact_search_text (\n\ttext_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tindex_def_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tbuild_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfragment_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfragment_locator TEXT NOT NULL, \n\tinput_hash VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tPRIMARY KEY (text_id), \n\tCONSTRAINT uq_pc_artifact_search_text_fragment UNIQUE (build_id, index_def_id, artifact_id, fragment_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family)\n)", + "CREATE INDEX ix_pc_artifact_search_text_search ON pc_artifact_search_text (scope_id, family, build_id, index_def_id, artifact_id)", + "CREATE FULLTEXT INDEX ix_pc_artifact_search_fts ON pc_artifact_search_text (searchable_text) WITH PARSER SPACE" + ], + "changes": [ + { + "pc_artifact_processing_binding_states": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "last_auto_wave_completed_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "last_schedule_checkpoint_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scan_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "scan_in_progress": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": [ + "literal", + "0" + ], + "charset": null, + "collation": null, + "extra": [] + }, + "scan_upper_pending_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "plugin_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "plugin_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "plugin_spec": { + "type": [ + "json", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "binding_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "desired_state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "phase": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "last_error": { + "type": [ + "varchar", + [ + "1024" + ] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_scan_generation": { + "expression": [ + "binary", + ">=", + [ + "column", + "scan_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_artifact_family_schemas": { + "columns": { + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "schema_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "declaration": { + "type": [ + "json", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "family", + null, + "asc" + ], + [ + "schema_version", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + } + }, + { + "pc_artifact_vector_spaces": { + "columns": { + "space_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "profile_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "profile_json": { + "type": [ + "json", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dimension": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "distance": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "normalization": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "storage_format_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "backend": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "physical_table": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "provision_state": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "space_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_artifact_vector_dimension": { + "expression": [ + "binary", + ">", + [ + "column", + "dimension" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_artifact_vector_space": { + "columns": [ + [ + "backend", + null, + "asc" + ], + [ + "profile_fingerprint", + null, + "asc" + ], + [ + "storage_format_version", + null, + "asc" + ] + ], + "unique": true + } + } + } + }, + { + "pc_artifact_vector_indexes": { + "columns": { + "index_def_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "index_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "schema_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "projection_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "space_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_kind": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "projection_spec": { + "type": [ + "json", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "definition_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "index_def_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "family", + "schema_version" + ], + "referred_schema": null, + "referred_table": "pc_artifact_family_schemas", + "referred_columns": [ + "family", + "schema_version" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "space_id" + ], + "referred_schema": null, + "referred_table": "pc_artifact_vector_spaces", + "referred_columns": [ + "space_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": { + "uq_pc_artifact_index_definition": { + "columns": [ + [ + "family", + null, + "asc" + ], + [ + "index_name", + null, + "asc" + ], + [ + "schema_version", + null, + "asc" + ], + [ + "projection_version", + null, + "asc" + ], + [ + "space_id", + null, + "asc" + ], + [ + "definition_fingerprint", + null, + "asc" + ] + ], + "unique": true + } + } + } + }, + { + "pc_artifact_search_states": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "build_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "manifest": { + "type": [ + "json", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "cleanup_targets": { + "type": [ + "json", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "validation_token": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "start_seq": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "applied_seq": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "change_seq": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + } + }, + { + "pc_artifact_index_coverage": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "build_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "index_def_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fragment_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "projection_input_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "governance_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "build_id", + null, + "asc" + ], + [ + "index_def_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family" + ], + "referred_schema": null, + "referred_table": "pc_artifact_search_states", + "referred_columns": [ + "scope_id", + "family" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_index_fragment_count": { + "expression": [ + "binary", + ">=", + [ + "column", + "fragment_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_artifact_index_changes": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "seq": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "schema_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "governance_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "seq", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family" + ], + "referred_schema": null, + "referred_table": "pc_artifact_search_states", + "referred_columns": [ + "scope_id", + "family" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + } + }, + { + "pc_artifact_search_text": { + "columns": { + "text_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "index_def_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "build_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fragment_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fragment_locator": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "governance_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "text_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "family" + ], + "referred_schema": null, + "referred_table": "pc_artifact_search_states", + "referred_columns": [ + "scope_id", + "family" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": { + "uq_pc_artifact_search_text_fragment": { + "columns": [ + [ + "build_id", + null, + "asc" + ], + [ + "index_def_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "fragment_id", + null, + "asc" + ] + ], + "unique": true + } + } + } + }, + { + "pc_artifact_search_text": { + "columns": { + "text_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "index_def_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "build_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fragment_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fragment_locator": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "governance_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "text_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "family" + ], + "referred_schema": null, + "referred_table": "pc_artifact_search_states", + "referred_columns": [ + "scope_id", + "family" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": { + "uq_pc_artifact_search_text_fragment": { + "columns": [ + [ + "build_id", + null, + "asc" + ], + [ + "index_def_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "fragment_id", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_artifact_search_text_search": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "build_id", + null, + "asc" + ], + [ + "index_def_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + { + "pc_artifact_search_text": { + "columns": { + "text_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "index_def_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "build_id": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fragment_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fragment_locator": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "governance_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "text_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "family" + ], + "referred_schema": null, + "referred_table": "pc_artifact_search_states", + "referred_columns": [ + "scope_id", + "family" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": { + "uq_pc_artifact_search_text_fragment": { + "columns": [ + [ + "build_id", + null, + "asc" + ], + [ + "index_def_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "fragment_id", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_artifact_search_text_search": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "build_id", + null, + "asc" + ], + [ + "index_def_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_artifact_search_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + } + } + ] + } + } +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_001.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_001.json new file mode 100644 index 0000000000..4789753600 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_001.json @@ -0,0 +1,765 @@ +{ + "statements": [ + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_id VARCHAR(128)", + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_family VARCHAR(128)", + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_spec JSON", + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN binding_generation BIGINT DEFAULT '0' NOT NULL", + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN desired_state VARCHAR(16)", + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN phase VARCHAR(16)", + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN last_error VARCHAR(1024)", + "CREATE TABLE pc_artifact_family_schemas (\n\tfamily VARCHAR(128) NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tdeclaration JSON NOT NULL, \n\tPRIMARY KEY (family, schema_version)\n)", + "CREATE TABLE pc_artifact_vector_spaces (\n\tspace_id VARCHAR(32) NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tprofile_json JSON NOT NULL, \n\tdimension INTEGER NOT NULL, \n\tdistance VARCHAR(16) NOT NULL, \n\tnormalization VARCHAR(16) NOT NULL, \n\tstorage_format_version VARCHAR(64) NOT NULL, \n\tbackend VARCHAR(32) NOT NULL, \n\tphysical_table VARCHAR(128) NOT NULL, \n\tprovision_state VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (space_id), \n\tCONSTRAINT uq_pc_artifact_vector_space UNIQUE (backend, profile_fingerprint, storage_format_version), \n\tCONSTRAINT ck_pc_artifact_vector_dimension CHECK (dimension > 0)\n)", + "CREATE TABLE pc_artifact_vector_indexes (\n\tindex_def_id VARCHAR(64) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tindex_name VARCHAR(128) NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tprojection_version VARCHAR(128) NOT NULL, \n\tspace_id VARCHAR(32), \n\tsource_kind VARCHAR(16) NOT NULL, \n\tprojection_spec JSON NOT NULL, \n\tdefinition_fingerprint VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (index_def_id), \n\tCONSTRAINT uq_pc_artifact_index_definition UNIQUE (family, index_name, schema_version, projection_version, space_id, definition_fingerprint), \n\tFOREIGN KEY(space_id) REFERENCES pc_artifact_vector_spaces (space_id), \n\tFOREIGN KEY(family, schema_version) REFERENCES pc_artifact_family_schemas (family, schema_version)\n)", + "CREATE TABLE pc_artifact_search_states (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tbuild_id VARCHAR(32), \n\tstate VARCHAR(16) NOT NULL, \n\tmanifest JSON NOT NULL, \n\tcleanup_targets JSON NOT NULL, \n\tvalidation_token VARCHAR(32), \n\tstart_seq BIGINT DEFAULT '0' NOT NULL, \n\tapplied_seq BIGINT DEFAULT '0' NOT NULL, \n\tchange_seq BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family)\n)", + "CREATE TABLE pc_artifact_index_coverage (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tbuild_id VARCHAR(32) NOT NULL, \n\tindex_def_id VARCHAR(64) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tfragment_count INTEGER NOT NULL, \n\tprojection_input_hash VARCHAR(64) NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, family, build_id, index_def_id, artifact_id), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family), \n\tCONSTRAINT ck_pc_artifact_index_fragment_count CHECK (fragment_count >= 0)\n)", + "CREATE TABLE pc_artifact_index_changes (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tseq BIGINT NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER, \n\toperation VARCHAR(16) NOT NULL, \n\tschema_version VARCHAR(128), \n\tgovernance_generation BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, family, seq), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family)\n)", + "CREATE TABLE pc_artifact_vectors (\n\tvector_id VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tindex_def_id VARCHAR(64) NOT NULL, \n\tbuild_id VARCHAR(32) NOT NULL, \n\tfragment_id VARCHAR(128) NOT NULL, \n\tfragment_locator TEXT NOT NULL, \n\tinput_hash VARCHAR(64) NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tspace_id VARCHAR(32) NOT NULL, \n\tdimension INTEGER NOT NULL, \n\tembedding BLOB NOT NULL, \n\tPRIMARY KEY (vector_id), \n\tCONSTRAINT uq_pc_artifact_vectors_fragment UNIQUE (build_id, index_def_id, artifact_id, fragment_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family), \n\tCONSTRAINT ck_pc_artifact_vectors_encoding CHECK (dimension > 0 AND length(embedding) = 4 * dimension)\n)", + "CREATE INDEX ix_pc_artifact_vectors_search ON pc_artifact_vectors (scope_id, family, build_id, space_id, index_def_id, artifact_id)", + "CREATE TABLE pc_artifact_search_text (\n\ttext_id VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tindex_def_id VARCHAR(64) NOT NULL, \n\tbuild_id VARCHAR(32) NOT NULL, \n\tfragment_id VARCHAR(128) NOT NULL, \n\tfragment_locator TEXT NOT NULL, \n\tinput_hash VARCHAR(64) NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tPRIMARY KEY (text_id), \n\tCONSTRAINT uq_pc_artifact_search_text_fragment UNIQUE (build_id, index_def_id, artifact_id, fragment_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family)\n)", + "CREATE INDEX ix_pc_artifact_search_text_search ON pc_artifact_search_text (scope_id, family, build_id, index_def_id, artifact_id)", + "CREATE VIRTUAL TABLE pc_artifact_search_fts USING fts5(text_id UNINDEXED, scope_id UNINDEXED, family UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED, build_id UNINDEXED, index_def_id UNINDEXED, searchable_text, tokenize='unicode61')", + "CREATE TRIGGER pc_artifact_search_fts_delete AFTER delete ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id; END", + "CREATE TRIGGER pc_artifact_search_fts_insert AFTER insert ON pc_artifact_search_text BEGIN INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END", + "CREATE TRIGGER pc_artifact_search_fts_update AFTER update ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id;INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END" + ], + "layout": [ + [ + "index", + "ix_pc_artifact_search_text_search", + "pc_artifact_search_text", + "CREATE INDEX ix_pc_artifact_search_text_search ON pc_artifact_search_text (scope_id, family, build_id, index_def_id, artifact_id)" + ], + [ + "index", + "ix_pc_artifact_tags_family_key", + "pc_artifact_tags", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)" + ], + [ + "index", + "ix_pc_artifact_tags_key", + "pc_artifact_tags", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ], + [ + "index", + "ix_pc_artifact_vectors_search", + "pc_artifact_vectors", + "CREATE INDEX ix_pc_artifact_vectors_search ON pc_artifact_vectors (scope_id, family, build_id, space_id, index_def_id, artifact_id)" + ], + [ + "index", + "ix_pc_context_bootstrap_receipts_created_at", + "pc_context_bootstrap_receipts", + "CREATE INDEX ix_pc_context_bootstrap_receipts_created_at ON pc_context_bootstrap_receipts (created_at)" + ], + [ + "index", + "ix_pc_decision_observations_operation", + "pc_decision_observations", + "CREATE INDEX ix_pc_decision_observations_operation ON pc_decision_observations (scope_id, operation_id, created_at)" + ], + [ + "index", + "ix_pc_decision_observations_scope_created", + "pc_decision_observations", + "CREATE INDEX ix_pc_decision_observations_scope_created ON pc_decision_observations (scope_id, created_at, observation_id)" + ], + [ + "index", + "ix_pc_dream_dispatch", + "pc_dream_runs", + "CREATE INDEX ix_pc_dream_dispatch ON pc_dream_runs (scope_id, operation, status, request_generation)" + ], + [ + "index", + "ix_pc_dream_list", + "pc_dream_runs", + "CREATE INDEX ix_pc_dream_list ON pc_dream_runs (scope_id, accepted_at, run_id)" + ], + [ + "index", + "ix_pc_dream_runs_proposal_fingerprint", + "pc_dream_runs", + "CREATE INDEX ix_pc_dream_runs_proposal_fingerprint ON pc_dream_runs (scope_id, principal_key, proposal_fingerprint)" + ], + [ + "index", + "ix_pc_processing_intent_binding_sequence", + "pc_artifact_processing_intents", + "CREATE INDEX ix_pc_processing_intent_binding_sequence ON pc_artifact_processing_intents (binding_name, pending_sequence)" + ], + [ + "index", + "ix_pc_recurrence_observation_key", + "pc_recurrence_observation", + "CREATE INDEX ix_pc_recurrence_observation_key ON pc_recurrence_observation (scope_id, signature_key_hash)" + ], + [ + "index", + "ix_pc_recurrence_observation_outcome", + "pc_recurrence_observation", + "CREATE INDEX ix_pc_recurrence_observation_outcome ON pc_recurrence_observation (scope_id, task_outcome_source_type, task_outcome_source_id, task_outcome_position)" + ], + [ + "index", + "ix_pc_recurrence_observation_revision", + "pc_recurrence_observation", + "CREATE INDEX ix_pc_recurrence_observation_revision ON pc_recurrence_observation (scope_id, family, artifact_id, revision)" + ], + [ + "table", + "pc_access_audit", + "pc_access_audit", + "CREATE TABLE pc_access_audit (\n\tcursor INTEGER NOT NULL, \n\tevent_id VARCHAR(64) NOT NULL, \n\toccurred_at VARCHAR(32) NOT NULL, \n\trequest_id VARCHAR(128), \n\ttransport VARCHAR(16) NOT NULL, \n\toperation VARCHAR(128) NOT NULL, \n\tprincipal_type VARCHAR(16) NOT NULL, \n\tprincipal_id VARCHAR(255) NOT NULL, \n\tprincipal_description TEXT, \n\tactor_type VARCHAR(16), \n\tactor_id VARCHAR(255), \n\tactor_description TEXT, \n\taction VARCHAR(64) NOT NULL, \n\tresource_type VARCHAR(16) NOT NULL, \n\tdeployment_id VARCHAR(128), \n\tscope_id VARCHAR(256), \n\tfamily VARCHAR(128), \n\tartifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\tallowed BOOLEAN NOT NULL, \n\treason_code VARCHAR(64) NOT NULL, \n\tpolicy_revision VARCHAR(64), \n\tmatched_subject_type VARCHAR(16), \n\tmatched_subject_id VARCHAR(255), \n\tmatched_subject_description TEXT, \n\tbinding_id VARCHAR(64), \n\ttarget_type VARCHAR(16), \n\ttarget_id VARCHAR(255), \n\ttarget_description TEXT, \n\trole VARCHAR(32), \n\texpected_version INTEGER, \n\tresult_version INTEGER, \n\tPRIMARY KEY (cursor), \n\tUNIQUE (event_id)\n)" + ], + [ + "table", + "pc_access_idempotency", + "pc_access_idempotency", + "CREATE TABLE pc_access_idempotency (\n\tactor_id VARCHAR(255) NOT NULL, \n\tidempotency_key_hash VARCHAR(64) NOT NULL, \n\toperation VARCHAR(64) NOT NULL, \n\tpayload_hash VARCHAR(64) NOT NULL, \n\tresult_binding_id VARCHAR(64), \n\tsecondary_binding_id VARCHAR(64), \n\tPRIMARY KEY (actor_id, idempotency_key_hash), \n\tCONSTRAINT uq_pc_access_idempotency_actor_key UNIQUE (actor_id, idempotency_key_hash)\n)" + ], + [ + "table", + "pc_access_owner_recovery", + "pc_access_owner_recovery", + "CREATE TABLE pc_access_owner_recovery (\n\toperation_id VARCHAR(64) NOT NULL, \n\tdeployment_id VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tcompleted BOOLEAN NOT NULL, \n\tpayload TEXT NOT NULL, \n\tPRIMARY KEY (operation_id), \n\tCONSTRAINT uq_pc_access_owner_recovery_pending UNIQUE (deployment_id, completed, operation_id)\n)" + ], + [ + "table", + "pc_access_owners", + "pc_access_owners", + "CREATE TABLE pc_access_owners (\n\towner_kind VARCHAR(16) NOT NULL, \n\tobject_key_hash VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128), \n\tcandidate_id VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\towner_type VARCHAR(16) NOT NULL, \n\towner_id VARCHAR(255) NOT NULL, \n\towner_description TEXT, \n\testablished_at VARCHAR(32), \n\tpolicy_revision VARCHAR(64), \n\tidempotency_key VARCHAR(255) NOT NULL, \n\tPRIMARY KEY (owner_kind, object_key_hash), \n\tCONSTRAINT ck_pc_access_owners_kind CHECK ((owner_kind = 'artifact' AND artifact_id IS NOT NULL AND candidate_id IS NULL AND target_artifact_id IS NULL AND established_at IS NOT NULL AND policy_revision IS NOT NULL) OR (owner_kind = 'candidate' AND candidate_id IS NOT NULL AND artifact_id IS NULL AND selector_type IS NULL AND selector_entry_id IS NULL AND established_at IS NULL AND policy_revision IS NULL))\n)" + ], + [ + "table", + "pc_access_relationship_heads", + "pc_access_relationship_heads", + "CREATE TABLE pc_access_relationship_heads (\n\tname VARCHAR(32) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tPRIMARY KEY (name), \n\tCONSTRAINT ck_pc_access_relationship_heads_revision_nonnegative CHECK (revision >= 0)\n)" + ], + [ + "table", + "pc_access_relationships", + "pc_access_relationships", + "CREATE TABLE pc_access_relationships (\n\tbinding_id VARCHAR(64) NOT NULL, \n\tsubject_type VARCHAR(16) NOT NULL, \n\tsubject_id VARCHAR(255) NOT NULL, \n\tsubject_description TEXT, \n\tresource_key_hash VARCHAR(64) NOT NULL, \n\tresource_type VARCHAR(16) NOT NULL, \n\tdeployment_id VARCHAR(128), \n\tscope_id VARCHAR(256), \n\tfamily VARCHAR(128), \n\tartifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\trole VARCHAR(32) NOT NULL, \n\tsingleton_key VARCHAR(64), \n\tgranted_by_type VARCHAR(16) NOT NULL, \n\tgranted_by_id VARCHAR(255) NOT NULL, \n\tgranted_by_description TEXT, \n\treason TEXT, \n\tcreated_at VARCHAR(32) NOT NULL, \n\texpires_at VARCHAR(32), \n\tstate VARCHAR(16) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tpolicy_revision VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(255) NOT NULL, \n\trevoked_at VARCHAR(32), \n\trevoked_by_type VARCHAR(16), \n\trevoked_by_id VARCHAR(255), \n\trevoked_by_description TEXT, \n\tPRIMARY KEY (binding_id), \n\tCONSTRAINT ck_pc_access_relationships_version_positive CHECK (version > 0), \n\tUNIQUE (singleton_key)\n)" + ], + [ + "table", + "pc_agent_skill_targets", + "pc_agent_skill_targets", + "CREATE TABLE pc_agent_skill_targets (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_id VARCHAR(64) NOT NULL, \n\tdisplay_name VARCHAR(128) NOT NULL, \n\tagent_kind VARCHAR(32) NOT NULL, \n\tinstallation_scope VARCHAR(16) NOT NULL, \n\tdelivery_mode VARCHAR(16) NOT NULL, \n\tinstallation_id VARCHAR(128), \n\tstate VARCHAR(16) NOT NULL, \n\tenrollment_token_digest VARCHAR(64), \n\tenrollment_expires_at DATETIME, \n\tcredential_subject VARCHAR(128), \n\tcredential_verifier VARCHAR(64), \n\treceiver_version VARCHAR(64), \n\tenvironment_fingerprint VARCHAR(64), \n\tmachine_hostname VARCHAR(255), \n\tworkspace_name VARCHAR(128), \n\tlast_seen_at DATETIME, \n\tgeneration BIGINT NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, target_id), \n\tCONSTRAINT uq_pc_agent_skill_targets_installation UNIQUE (scope_id, agent_kind, installation_scope, installation_id), \n\tCONSTRAINT uq_pc_agent_skill_targets_enrollment_token UNIQUE (enrollment_token_digest), \n\tCONSTRAINT uq_pc_agent_skill_targets_credential_subject UNIQUE (credential_subject), \n\tCONSTRAINT uq_pc_agent_skill_targets_credential_verifier UNIQUE (credential_verifier), \n\tCONSTRAINT ck_pc_agent_skill_targets_agent_kind CHECK (agent_kind IN ('codex', 'claude_code')), \n\tCONSTRAINT ck_pc_agent_skill_targets_installation_scope CHECK (installation_scope IN ('project')), \n\tCONSTRAINT ck_pc_agent_skill_targets_delivery_mode CHECK (delivery_mode = 'agent_pull'), \n\tCONSTRAINT ck_pc_agent_skill_targets_state CHECK (state IN ('pending', 'active', 'revoked')), \n\tCONSTRAINT ck_pc_agent_skill_targets_state_payload CHECK ((state = 'pending' AND enrollment_token_digest IS NOT NULL AND enrollment_expires_at IS NOT NULL AND installation_id IS NULL AND credential_subject IS NULL AND credential_verifier IS NULL) OR (state = 'active' AND enrollment_token_digest IS NULL AND enrollment_expires_at IS NULL AND installation_id IS NOT NULL AND credential_subject IS NOT NULL AND credential_verifier IS NOT NULL) OR (state = 'revoked' AND enrollment_token_digest IS NULL AND enrollment_expires_at IS NULL AND credential_verifier IS NULL)), \n\tCONSTRAINT ck_pc_agent_skill_targets_generation_nonnegative CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_artifact_family_schemas", + "pc_artifact_family_schemas", + "CREATE TABLE pc_artifact_family_schemas (\n\tfamily VARCHAR(128) NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tdeclaration JSON NOT NULL, \n\tPRIMARY KEY (family, schema_version)\n)" + ], + [ + "table", + "pc_artifact_fts", + "pc_artifact_fts", + "CREATE VIRTUAL TABLE pc_artifact_fts USING fts5(\n scope_id UNINDEXED,\n family UNINDEXED,\n artifact_id UNINDEXED,\n revision UNINDEXED,\n searchable_text,\n tokenize='unicode61'\n)" + ], + [ + "table", + "pc_artifact_fts_config", + "pc_artifact_fts_config", + "CREATE TABLE 'pc_artifact_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_artifact_fts_content", + "pc_artifact_fts_content", + "CREATE TABLE 'pc_artifact_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4)" + ], + [ + "table", + "pc_artifact_fts_data", + "pc_artifact_fts_data", + "CREATE TABLE 'pc_artifact_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_artifact_fts_docsize", + "pc_artifact_fts_docsize", + "CREATE TABLE 'pc_artifact_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_artifact_fts_idx", + "pc_artifact_fts_idx", + "CREATE TABLE 'pc_artifact_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_artifact_handles", + "pc_artifact_handles", + "CREATE TABLE pc_artifact_handles (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\thandle_key VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id, family, handle_key), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_handles_revision_positive CHECK (revision > 0)\n)" + ], + [ + "table", + "pc_artifact_heads", + "pc_artifact_heads", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text TEXT, \n\tlifecycle_state VARCHAR(16) DEFAULT 'active' NOT NULL, \n\treplacement_artifact_id VARCHAR(128), \n\tmerged_into_id VARCHAR(128), \n\tgovernance_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated'), \n\tCONSTRAINT ck_pc_artifact_heads_merge_target CHECK (merged_into_id IS NULL OR (lifecycle_state = 'deprecated' AND merged_into_id <> artifact_id AND replacement_artifact_id IS NULL))\n)" + ], + [ + "table", + "pc_artifact_index_changes", + "pc_artifact_index_changes", + "CREATE TABLE pc_artifact_index_changes (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tseq BIGINT NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER, \n\toperation VARCHAR(16) NOT NULL, \n\tschema_version VARCHAR(128), \n\tgovernance_generation BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, family, seq), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family)\n)" + ], + [ + "table", + "pc_artifact_index_coverage", + "pc_artifact_index_coverage", + "CREATE TABLE pc_artifact_index_coverage (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tbuild_id VARCHAR(32) NOT NULL, \n\tindex_def_id VARCHAR(64) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tfragment_count INTEGER NOT NULL, \n\tprojection_input_hash VARCHAR(64) NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, family, build_id, index_def_id, artifact_id), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family), \n\tCONSTRAINT ck_pc_artifact_index_fragment_count CHECK (fragment_count >= 0)\n)" + ], + [ + "table", + "pc_artifact_lineage_artifacts", + "pc_artifact_lineage_artifacts", + "CREATE TABLE pc_artifact_lineage_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tupstream_family VARCHAR(128) NOT NULL, \n\tupstream_artifact_id VARCHAR(128) NOT NULL, \n\tupstream_revision INTEGER NOT NULL, \n\tis_merge_input BOOLEAN DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision, ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, upstream_family, upstream_artifact_id, upstream_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_artifact_lineage_sources", + "pc_artifact_lineage_sources", + "CREATE TABLE pc_artifact_lineage_sources (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision, ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, source_type, source_id) REFERENCES pc_sources (scope_id, source_type, source_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_artifact_processing_auto_wave_targets", + "pc_artifact_processing_auto_wave_targets", + "CREATE TABLE pc_artifact_processing_auto_wave_targets (\n\twave_id VARCHAR(36) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tcompleted BOOLEAN DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (wave_id, binding_name, scope_id), \n\tCONSTRAINT ck_pc_artifact_processing_auto_wave_target_source_positive CHECK (source_through >= 1)\n)" + ], + [ + "table", + "pc_artifact_processing_binding_states", + "pc_artifact_processing_binding_states", + "CREATE TABLE pc_artifact_processing_binding_states (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tlast_auto_wave_completed_at DATETIME, \n\tlast_schedule_checkpoint_at DATETIME, \n\tscan_generation BIGINT DEFAULT '0' NOT NULL, \n\tscan_in_progress BOOLEAN DEFAULT '0' NOT NULL, \n\tscan_upper_pending_sequence BIGINT, plugin_id VARCHAR(128), plugin_family VARCHAR(128), plugin_spec JSON, binding_generation BIGINT DEFAULT '0' NOT NULL, desired_state VARCHAR(16), phase VARCHAR(16), last_error VARCHAR(1024), \n\tPRIMARY KEY (binding_name), \n\tCONSTRAINT ck_pc_processing_scan_generation CHECK (scan_generation >= 0)\n)" + ], + [ + "table", + "pc_artifact_processing_intents", + "pc_artifact_processing_intents", + "CREATE TABLE \"pc_artifact_processing_intents\" (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tpending_sequence BIGINT NOT NULL, \n\tdirty_generation BIGINT DEFAULT '0' NOT NULL, \n\tclean_generation BIGINT DEFAULT '0' NOT NULL, \n\trequested_generation BIGINT DEFAULT '0' NOT NULL, \n\thandled_generation BIGINT DEFAULT '0' NOT NULL, \n\tlast_auto_scan_generation BIGINT DEFAULT '0' NOT NULL, \n\tconsecutive_dream_attempts BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_processing_intent_sequence CHECK (pending_sequence > 0), \n\tCONSTRAINT ck_pc_processing_intent_dirty CHECK (clean_generation >= 0 AND clean_generation <= dirty_generation), \n\tCONSTRAINT ck_pc_processing_intent_request CHECK (handled_generation >= 0 AND handled_generation <= requested_generation), \n\tCONSTRAINT ck_pc_processing_intent_scan CHECK (last_auto_scan_generation >= 0), \n\tUNIQUE (pending_sequence)\n)" + ], + [ + "table", + "pc_artifact_processing_leases", + "pc_artifact_processing_leases", + "CREATE TABLE pc_artifact_processing_leases (\n\tsupervisor_group VARCHAR(64) NOT NULL, \n\tholder_id VARCHAR(36) NOT NULL, \n\tsupervisor_generation BIGINT NOT NULL, \n\tlease_expires_at DATETIME, \n\tPRIMARY KEY (supervisor_group), \n\tCONSTRAINT ck_pc_artifact_processing_leases_generation_positive CHECK (supervisor_generation > 0)\n)" + ], + [ + "table", + "pc_artifact_processing_migration_receipts", + "pc_artifact_processing_migration_receipts", + "CREATE TABLE pc_artifact_processing_migration_receipts (\n\tmigration_id VARCHAR(64) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_snapshot BLOB NOT NULL, \n\trequested_generation BIGINT NOT NULL, \n\tdirty_generation BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tPRIMARY KEY (migration_id, binding_name, scope_id)\n)" + ], + [ + "table", + "pc_artifact_processing_pending", + "pc_artifact_processing_pending", + "CREATE TABLE pc_artifact_processing_pending (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tflush_generation BIGINT DEFAULT '0' NOT NULL, \n\thandled_flush_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_artifact_processing_pending_source_positive CHECK (source_through >= 1), \n\tCONSTRAINT ck_pc_artifact_processing_pending_flush_nonnegative CHECK (flush_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_processing_pending_handled_nonnegative CHECK (handled_flush_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_processing_pending_handled_not_ahead CHECK (handled_flush_generation <= flush_generation)\n)" + ], + [ + "table", + "pc_artifact_processing_schema", + "pc_artifact_processing_schema", + "CREATE TABLE pc_artifact_processing_schema (\n\tsingleton INTEGER NOT NULL, \n\tschema_version INTEGER NOT NULL, \n\tsource_version INTEGER DEFAULT '1490' NOT NULL, \n\tphase VARCHAR(32) NOT NULL, \n\tmigration_id VARCHAR(64) NOT NULL, \n\tconfig_manifest TEXT NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_processing_schema_singleton CHECK (singleton = 1)\n)" + ], + [ + "table", + "pc_artifact_processing_sequences", + "pc_artifact_processing_sequences", + "CREATE TABLE pc_artifact_processing_sequences (\n\tsingleton INTEGER NOT NULL, \n\tsequence BIGINT NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_processing_sequence CHECK (singleton = 1 AND sequence >= 0)\n)" + ], + [ + "table", + "pc_artifact_publications", + "pc_artifact_publications", + "CREATE TABLE pc_artifact_publications (\n\ttarget_scope_id VARCHAR(256) NOT NULL, \n\ttarget_family VARCHAR(128) NOT NULL, \n\ttarget_artifact_id VARCHAR(128) NOT NULL, \n\ttarget_revision INTEGER NOT NULL, \n\tsource_scope_id VARCHAR(256) NOT NULL, \n\tsource_family VARCHAR(128) NOT NULL, \n\tsource_artifact_id VARCHAR(128) NOT NULL, \n\tsource_revision INTEGER NOT NULL, \n\tcontent_digest VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (target_scope_id, target_family, target_artifact_id, target_revision), \n\tFOREIGN KEY(target_scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tFOREIGN KEY(source_scope_id, source_family, source_artifact_id, source_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT uq_pc_artifact_publications_request UNIQUE (target_scope_id, idempotency_key)\n)" + ], + [ + "table", + "pc_artifact_search_fts", + "pc_artifact_search_fts", + "CREATE VIRTUAL TABLE pc_artifact_search_fts USING fts5(text_id UNINDEXED, scope_id UNINDEXED, family UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED, build_id UNINDEXED, index_def_id UNINDEXED, searchable_text, tokenize='unicode61')" + ], + [ + "table", + "pc_artifact_search_fts_config", + "pc_artifact_search_fts_config", + "CREATE TABLE 'pc_artifact_search_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_artifact_search_fts_content", + "pc_artifact_search_fts_content", + "CREATE TABLE 'pc_artifact_search_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4, c5, c6, c7)" + ], + [ + "table", + "pc_artifact_search_fts_data", + "pc_artifact_search_fts_data", + "CREATE TABLE 'pc_artifact_search_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_artifact_search_fts_docsize", + "pc_artifact_search_fts_docsize", + "CREATE TABLE 'pc_artifact_search_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_artifact_search_fts_idx", + "pc_artifact_search_fts_idx", + "CREATE TABLE 'pc_artifact_search_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_artifact_search_states", + "pc_artifact_search_states", + "CREATE TABLE pc_artifact_search_states (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tbuild_id VARCHAR(32), \n\tstate VARCHAR(16) NOT NULL, \n\tmanifest JSON NOT NULL, \n\tcleanup_targets JSON NOT NULL, \n\tvalidation_token VARCHAR(32), \n\tstart_seq BIGINT DEFAULT '0' NOT NULL, \n\tapplied_seq BIGINT DEFAULT '0' NOT NULL, \n\tchange_seq BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family)\n)" + ], + [ + "table", + "pc_artifact_search_text", + "pc_artifact_search_text", + "CREATE TABLE pc_artifact_search_text (\n\ttext_id VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tindex_def_id VARCHAR(64) NOT NULL, \n\tbuild_id VARCHAR(32) NOT NULL, \n\tfragment_id VARCHAR(128) NOT NULL, \n\tfragment_locator TEXT NOT NULL, \n\tinput_hash VARCHAR(64) NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tPRIMARY KEY (text_id), \n\tCONSTRAINT uq_pc_artifact_search_text_fragment UNIQUE (build_id, index_def_id, artifact_id, fragment_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family)\n)" + ], + [ + "table", + "pc_artifact_tags", + "pc_artifact_tags", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\ttarget_type VARCHAR(12) NOT NULL, \n\ttarget_id VARCHAR(128) NOT NULL, \n\ttag_key_hash BLOB NOT NULL, \n\ttag_key VARCHAR(128) NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)" + ], + [ + "table", + "pc_artifact_vector_indexes", + "pc_artifact_vector_indexes", + "CREATE TABLE pc_artifact_vector_indexes (\n\tindex_def_id VARCHAR(64) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tindex_name VARCHAR(128) NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tprojection_version VARCHAR(128) NOT NULL, \n\tspace_id VARCHAR(32), \n\tsource_kind VARCHAR(16) NOT NULL, \n\tprojection_spec JSON NOT NULL, \n\tdefinition_fingerprint VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (index_def_id), \n\tCONSTRAINT uq_pc_artifact_index_definition UNIQUE (family, index_name, schema_version, projection_version, space_id, definition_fingerprint), \n\tFOREIGN KEY(space_id) REFERENCES pc_artifact_vector_spaces (space_id), \n\tFOREIGN KEY(family, schema_version) REFERENCES pc_artifact_family_schemas (family, schema_version)\n)" + ], + [ + "table", + "pc_artifact_vector_spaces", + "pc_artifact_vector_spaces", + "CREATE TABLE pc_artifact_vector_spaces (\n\tspace_id VARCHAR(32) NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tprofile_json JSON NOT NULL, \n\tdimension INTEGER NOT NULL, \n\tdistance VARCHAR(16) NOT NULL, \n\tnormalization VARCHAR(16) NOT NULL, \n\tstorage_format_version VARCHAR(64) NOT NULL, \n\tbackend VARCHAR(32) NOT NULL, \n\tphysical_table VARCHAR(128) NOT NULL, \n\tprovision_state VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (space_id), \n\tCONSTRAINT uq_pc_artifact_vector_space UNIQUE (backend, profile_fingerprint, storage_format_version), \n\tCONSTRAINT ck_pc_artifact_vector_dimension CHECK (dimension > 0)\n)" + ], + [ + "table", + "pc_artifact_vectors", + "pc_artifact_vectors", + "CREATE TABLE pc_artifact_vectors (\n\tvector_id VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tschema_version VARCHAR(128) NOT NULL, \n\tindex_def_id VARCHAR(64) NOT NULL, \n\tbuild_id VARCHAR(32) NOT NULL, \n\tfragment_id VARCHAR(128) NOT NULL, \n\tfragment_locator TEXT NOT NULL, \n\tinput_hash VARCHAR(64) NOT NULL, \n\tgovernance_generation BIGINT NOT NULL, \n\tspace_id VARCHAR(32) NOT NULL, \n\tdimension INTEGER NOT NULL, \n\tembedding BLOB NOT NULL, \n\tPRIMARY KEY (vector_id), \n\tCONSTRAINT uq_pc_artifact_vectors_fragment UNIQUE (build_id, index_def_id, artifact_id, fragment_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family) REFERENCES pc_artifact_search_states (scope_id, family), \n\tCONSTRAINT ck_pc_artifact_vectors_encoding CHECK (dimension > 0 AND length(embedding) = 4 * dimension)\n)" + ], + [ + "table", + "pc_artifacts", + "pc_artifacts", + "CREATE TABLE \"pc_artifacts\" (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tcreated_at DATETIME, \n\tcreated_by VARCHAR(2048), \n\trestored_from_revision INTEGER, \n\trollback_reason VARCHAR(1024), \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" + ], + [ + "table", + "pc_atomic_memory_current", + "pc_atomic_memory_current", + "CREATE TABLE pc_atomic_memory_current (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tstate_version BIGINT NOT NULL, \n\tcontent_hash VARCHAR(64) NOT NULL, \n\tprojection_format VARCHAR(64) NOT NULL, \n\tkind TEXT NOT NULL, \n\ttext TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\ttag_keys TEXT NOT NULL, \n\tembedding BLOB, \n\tprofile_fingerprint VARCHAR(64), \n\tembedding_input_hash VARCHAR(64), description VARCHAR(200), \n\tPRIMARY KEY (scope_id, artifact_id), \n\tCONSTRAINT ck_pc_atomic_memory_current_revision CHECK (revision > 0), \n\tCONSTRAINT ck_pc_atomic_memory_current_state_version CHECK (state_version >= 0), \n\tCONSTRAINT ck_pc_atomic_memory_current_embedding_metadata CHECK ((embedding IS NULL AND profile_fingerprint IS NULL AND embedding_input_hash IS NULL) OR (embedding IS NOT NULL AND profile_fingerprint IS NOT NULL AND embedding_input_hash IS NOT NULL))\n)" + ], + [ + "table", + "pc_atomic_memory_current_fts", + "pc_atomic_memory_current_fts", + "CREATE VIRTUAL TABLE pc_atomic_memory_current_fts USING fts5(scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED, state_version UNINDEXED, content_hash UNINDEXED, projection_format UNINDEXED, kind UNINDEXED, text UNINDEXED, searchable_text, tag_keys UNINDEXED, identity_token, tokenize='unicode61')" + ], + [ + "table", + "pc_atomic_memory_current_fts_config", + "pc_atomic_memory_current_fts_config", + "CREATE TABLE 'pc_atomic_memory_current_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_atomic_memory_current_fts_content", + "pc_atomic_memory_current_fts_content", + "CREATE TABLE 'pc_atomic_memory_current_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10)" + ], + [ + "table", + "pc_atomic_memory_current_fts_data", + "pc_atomic_memory_current_fts_data", + "CREATE TABLE 'pc_atomic_memory_current_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_atomic_memory_current_fts_docsize", + "pc_atomic_memory_current_fts_docsize", + "CREATE TABLE 'pc_atomic_memory_current_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_atomic_memory_current_fts_idx", + "pc_atomic_memory_current_fts_idx", + "CREATE TABLE 'pc_atomic_memory_current_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_candidate_heads", + "pc_candidate_heads", + "CREATE TABLE \"pc_candidate_heads\" (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tstatus VARCHAR(16) NOT NULL, \n\tcandidate_kind VARCHAR(16) DEFAULT 'artifact' NOT NULL, \n\tresult_payload BLOB, \n\tresult_family VARCHAR(128), \n\tresult_artifact_id VARCHAR(128), \n\tresult_revision INTEGER, \n\tdecision_reason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, candidate_id, version) REFERENCES pc_candidate_versions (scope_id, candidate_id, version) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, result_family, result_artifact_id, result_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_candidate_kind CHECK (candidate_kind IN ('artifact', 'tag')), \n\tCONSTRAINT ck_pc_candidate_heads_status CHECK (status IN ('pending', 'approved', 'rejected')), \n\tCONSTRAINT ck_pc_candidate_heads_terminal_result CHECK ((status = 'approved' AND candidate_kind = 'artifact' AND result_payload IS NULL AND result_family IS NOT NULL AND result_artifact_id IS NOT NULL AND result_revision IS NOT NULL AND result_revision > 0 AND decision_reason IS NULL) OR (status = 'approved' AND candidate_kind = 'tag' AND result_payload IS NOT NULL AND result_family IS NULL AND result_artifact_id IS NULL AND result_revision IS NULL AND decision_reason IS NULL) OR (status = 'rejected' AND result_payload IS NULL AND result_family IS NULL AND result_artifact_id IS NULL AND result_revision IS NULL AND decision_reason IS NOT NULL) OR (status = 'pending' AND result_payload IS NULL AND result_family IS NULL AND result_artifact_id IS NULL AND result_revision IS NULL AND decision_reason IS NULL))\n)" + ], + [ + "table", + "pc_candidate_versions", + "pc_candidate_versions", + "CREATE TABLE \"pc_candidate_versions\" (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tproposal BLOB NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\treason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)" + ], + [ + "table", + "pc_connector_checkpoints", + "pc_connector_checkpoints", + "CREATE TABLE pc_connector_checkpoints (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_id VARCHAR(256) NOT NULL, \n\tconnector_name VARCHAR(128) NOT NULL, \n\tconnector_version VARCHAR(128) NOT NULL, \n\tcheckpoint BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, binding_id)\n)" + ], + [ + "table", + "pc_context_bootstrap_receipts", + "pc_context_bootstrap_receipts", + "CREATE TABLE pc_context_bootstrap_receipts (\n\treceipt_id VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tevent_key VARCHAR(64), \n\trequest_digest VARCHAR(71) NOT NULL, \n\tintegration VARCHAR(64) NOT NULL, \n\tlifecycle VARCHAR(16) NOT NULL, \n\tprofile VARCHAR(64) NOT NULL, \n\tstate VARCHAR(16) NOT NULL, \n\treason VARCHAR(32), \n\tmax_bytes INTEGER NOT NULL, \n\tpackage_digest VARCHAR(71), \n\tcontent_bytes INTEGER NOT NULL, \n\ttruncated BOOLEAN NOT NULL, \n\titems BLOB NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tPRIMARY KEY (receipt_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_context_bootstrap_receipts_event UNIQUE (event_key), \n\tCONSTRAINT ck_pc_context_bootstrap_receipts_state CHECK (state IN ('pending', 'injected', 'skipped', 'failed')), \n\tCONSTRAINT ck_pc_context_bootstrap_receipts_budget CHECK (max_bytes >= 512 AND max_bytes <= 8192), \n\tCONSTRAINT ck_pc_context_bootstrap_receipts_content_bytes CHECK (content_bytes >= 0)\n)" + ], + [ + "table", + "pc_decision_observations", + "pc_decision_observations", + "CREATE TABLE pc_decision_observations (\n\tscope_id VARCHAR(256) NOT NULL, \n\tobservation_id VARCHAR(128) NOT NULL, \n\toperation_id VARCHAR(256) NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tconsumer VARCHAR(128) NOT NULL, \n\tpolicy_id VARCHAR(256) NOT NULL, \n\tpolicy_version VARCHAR(64) NOT NULL, \n\tmode VARCHAR(16) NOT NULL, \n\tprivacy_boundary VARCHAR(32) NOT NULL, \n\tcoverage VARCHAR(16) NOT NULL, \n\tverdict VARCHAR(16) NOT NULL, \n\tassessment_source VARCHAR(32) NOT NULL, \n\tfinal_action VARCHAR(128) NOT NULL, \n\tused_fallback BOOLEAN NOT NULL, \n\tfallback_reason VARCHAR(128), \n\tprovider_id VARCHAR(128), \n\tbackend_model_id VARCHAR(256), \n\tmodel_policy_id VARCHAR(256), \n\trequests BIGINT NOT NULL, \n\tinput_tokens BIGINT, \n\toutput_tokens BIGINT, \n\tlatency_ms FLOAT, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, observation_id), \n\tCONSTRAINT ck_pc_decision_observations_requests_nonnegative CHECK (requests >= 0), \n\tCONSTRAINT ck_pc_decision_observations_input_tokens_nonnegative CHECK (input_tokens IS NULL OR input_tokens >= 0), \n\tCONSTRAINT ck_pc_decision_observations_output_tokens_nonnegative CHECK (output_tokens IS NULL OR output_tokens >= 0), \n\tCONSTRAINT ck_pc_decision_observations_latency_nonnegative CHECK (latency_ms IS NULL OR latency_ms >= 0)\n)" + ], + [ + "table", + "pc_dream_runs", + "pc_dream_runs", + "CREATE TABLE \"pc_dream_runs\" (\n\tscope_id VARCHAR(256) NOT NULL, \n\trun_id VARCHAR(64) NOT NULL, \n\tprincipal_key VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(128) NOT NULL, \n\trequest_digest VARCHAR(71) NOT NULL, \n\tproposal_fingerprint VARCHAR(71), \n\toperation VARCHAR(32) NOT NULL, \n\tstatus VARCHAR(16) NOT NULL, \n\taccepted_at BIGINT NOT NULL, \n\tgeneration INTEGER NOT NULL, \n\trequest_generation BIGINT NOT NULL, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, run_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_dream_idempotency UNIQUE (scope_id, principal_key, idempotency_key), \n\tCONSTRAINT ck_pc_dream_generation CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_external_skill_registrations", + "pc_external_skill_registrations", + "CREATE TABLE pc_external_skill_registrations (\n\tscope_id VARCHAR(256) NOT NULL, \n\texternal_skill_id VARCHAR(128) NOT NULL, \n\tprovider VARCHAR(128) NOT NULL, \n\tagent_kind VARCHAR(128) NOT NULL, \n\thost_id VARCHAR(128) NOT NULL, \n\tinstallation_scope VARCHAR(16) NOT NULL, \n\tlocator VARCHAR(2000) NOT NULL, \n\tlocator_hash VARCHAR(64) NOT NULL, \n\tfingerprint VARCHAR(64) NOT NULL, \n\tname VARCHAR(128) NOT NULL, \n\tdescription VARCHAR(2000) NOT NULL, \n\tPRIMARY KEY (scope_id, external_skill_id), \n\tCONSTRAINT uq_pc_external_skill_registrations_binding UNIQUE (scope_id, provider, host_id, installation_scope, locator_hash), \n\tCONSTRAINT ck_pc_external_skill_registrations_scope CHECK (installation_scope IN ('user', 'project', 'plugin'))\n)" + ], + [ + "table", + "pc_lexical_projection_versions", + "pc_lexical_projection_versions", + "CREATE TABLE pc_lexical_projection_versions (\n\tprojection VARCHAR(128) NOT NULL, \n\tanalyzer_version VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (projection)\n)" + ], + [ + "table", + "pc_memory_artifact_archive", + "pc_memory_artifact_archive", + "CREATE TABLE pc_memory_artifact_archive (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tmemory_citations BLOB, \n\tmetadata BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" + ], + [ + "table", + "pc_memory_entry_heads", + "pc_memory_entry_heads", + "CREATE TABLE pc_memory_entry_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tmemory_artifact_id VARCHAR(128) NOT NULL, \n\thead_revision INTEGER NOT NULL, \n\tentry_id VARCHAR(128) NOT NULL, \n\tentry_version_id VARCHAR(128) NOT NULL, \n\tentry_content_hash VARCHAR(64) NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tPRIMARY KEY (scope_id, memory_artifact_id, entry_id), \n\tFOREIGN KEY(scope_id, memory_artifact_id, entry_id, entry_version_id) REFERENCES pc_memory_entry_versions (scope_id, memory_artifact_id, entry_id, entry_version_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_memory_entry_heads_revision_positive CHECK (head_revision > 0)\n)" + ], + [ + "table", + "pc_memory_entry_versions", + "pc_memory_entry_versions", + "CREATE TABLE pc_memory_entry_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tmemory_artifact_id VARCHAR(128) NOT NULL, \n\tentry_id VARCHAR(128) NOT NULL, \n\tentry_version_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tprevious_version_id VARCHAR(128), \n\tkind VARCHAR(128) NOT NULL, \n\ttext TEXT NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\tentry_content_hash VARCHAR(64) NOT NULL, \n\tcreated_in_revision INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id, memory_artifact_id, entry_version_id), \n\tCONSTRAINT uq_pc_memory_entry_versions_logical_version UNIQUE (scope_id, memory_artifact_id, entry_id, version), \n\tCONSTRAINT uq_pc_memory_entry_versions_identity UNIQUE (scope_id, memory_artifact_id, entry_id, entry_version_id), \n\tCONSTRAINT ck_pc_memory_entry_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_memory_entry_versions_revision_positive CHECK (created_in_revision > 0)\n)" + ], + [ + "table", + "pc_memory_source_windows", + "pc_memory_source_windows", + "CREATE TABLE pc_memory_source_windows (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\twindow_limit BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tCONSTRAINT ck_pc_memory_source_window_positive CHECK (source_through > 0 AND window_limit > 0)\n)" + ], + [ + "table", + "pc_model_usage_daily", + "pc_model_usage_daily", + "CREATE TABLE pc_model_usage_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpurpose VARCHAR(64) NOT NULL, \n\toperation VARCHAR(16) NOT NULL, \n\trequests BIGINT NOT NULL, \n\tinput_tokens BIGINT NOT NULL, \n\toutput_tokens BIGINT NOT NULL, \n\tinput_complete BOOLEAN NOT NULL, \n\toutput_complete BOOLEAN NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, purpose, operation), \n\tCONSTRAINT ck_pc_model_usage_daily_operation CHECK (operation IN ('generation', 'embedding')), \n\tCONSTRAINT ck_pc_model_usage_daily_requests_nonnegative CHECK (requests >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_input_nonnegative CHECK (input_tokens >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_output_nonnegative CHECK (output_tokens >= 0)\n)" + ], + [ + "table", + "pc_portable_restore_receipts", + "pc_portable_restore_receipts", + "CREATE TABLE pc_portable_restore_receipts (\n\tbundle_id VARCHAR(36) NOT NULL, \n\tformat_version INTEGER NOT NULL, \n\ttotal_digest VARCHAR(71) NOT NULL, \n\trecord_count BIGINT NOT NULL, \n\tstatus VARCHAR(32) NOT NULL, \n\tinserted BIGINT NOT NULL, \n\talready_present BIGINT NOT NULL, \n\tprojections_ready BOOLEAN NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (bundle_id), \n\tCONSTRAINT ck_pc_portable_restore_receipts_status CHECK (status IN ('authoritative_restored', 'ready')), \n\tCONSTRAINT ck_pc_portable_restore_receipts_record_count CHECK (record_count >= 0), \n\tCONSTRAINT ck_pc_portable_restore_receipts_inserted CHECK (inserted >= 0), \n\tCONSTRAINT ck_pc_portable_restore_receipts_existing CHECK (already_present >= 0)\n)" + ], + [ + "table", + "pc_profile_policies", + "pc_profile_policies", + "CREATE TABLE \"pc_profile_policies\" (\n\tscope_id VARCHAR(256) NOT NULL, \n\tgeneration_enabled BOOLEAN NOT NULL, \n\tactivation_mode VARCHAR(32) NOT NULL, \n\tpending_candidate_id VARCHAR(128), \n\tversion BIGINT NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, pending_candidate_id) REFERENCES pc_candidate_heads (scope_id, candidate_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_profile_policies_activation_mode CHECK (activation_mode IN ('automatic', 'review_required')), \n\tCONSTRAINT ck_pc_profile_policies_version_positive CHECK (version > 0)\n)" + ], + [ + "table", + "pc_recall_effort_daily", + "pc_recall_effort_daily", + "CREATE TABLE pc_recall_effort_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpolicy_id VARCHAR(128) NOT NULL, \n\tassessment VARCHAR(64) NOT NULL, \n\tpreparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_preparations_nonnegative CHECK (preparations >= 0), \n\trounds BIGINT NOT NULL CONSTRAINT ck_recall_effort_rounds_nonnegative CHECK (rounds >= 0), \n\texpanded_preparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_expanded_preparations_nonnegative CHECK (expanded_preparations >= 0), \n\tadmission_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_admission_expansions_nonnegative CHECK (admission_expansions >= 0), \n\tpolicy_floor_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_policy_floor_expansions_nonnegative CHECK (policy_floor_expansions >= 0), \n\tcandidate_round_samples BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidate_round_samples_nonnegative CHECK (candidate_round_samples >= 0), \n\tcandidates_assessed BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidates_assessed_nonnegative CHECK (candidates_assessed >= 0), \n\tfinal_candidate_pool BIGINT NOT NULL CONSTRAINT ck_recall_effort_final_candidate_pool_nonnegative CHECK (final_candidate_pool >= 0), \n\tadded_embeddings BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_embeddings_nonnegative CHECK (added_embeddings >= 0), \n\tadded_generation_calls BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_generation_calls_nonnegative CHECK (added_generation_calls >= 0), \n\ttruncated_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_truncated_items_nonnegative CHECK (truncated_items >= 0), \n\tdropped_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_items_nonnegative CHECK (dropped_items >= 0), \n\tdropped_below_min_bytes BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_below_min_bytes_nonnegative CHECK (dropped_below_min_bytes >= 0), \n\tdropped_no_fitting_truncation BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_no_fitting_truncation_nonnegative CHECK (dropped_no_fitting_truncation >= 0), \n\tPRIMARY KEY (scope_id, usage_date, policy_id, assessment), \n\tCONSTRAINT ck_pc_recall_effort_daily_drop_total CHECK (dropped_items = dropped_below_min_bytes + dropped_no_fitting_truncation)\n)" + ], + [ + "table", + "pc_recall_token_daily", + "pc_recall_token_daily", + "CREATE TABLE pc_recall_token_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\testimator_id VARCHAR(128) NOT NULL, \n\testimator_version VARCHAR(64) NOT NULL, \n\tpreparations BIGINT NOT NULL, \n\tready_preparations BIGINT NOT NULL, \n\tcomparable_preparations BIGINT NOT NULL, \n\tbaseline_tokens BIGINT NOT NULL, \n\trecalled_tokens BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, estimator_id, estimator_version), \n\tCONSTRAINT ck_pc_recall_token_daily_preparations_nonnegative CHECK (preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_nonnegative CHECK (ready_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_nonnegative CHECK (comparable_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_ready CHECK (comparable_preparations <= ready_preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_total CHECK (ready_preparations <= preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_baseline_nonnegative CHECK (baseline_tokens >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_recalled_nonnegative CHECK (recalled_tokens >= 0)\n)" + ], + [ + "table", + "pc_receipt_migration_review", + "pc_receipt_migration_review", + "CREATE TABLE pc_receipt_migration_review (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\treason VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, source_id)\n)" + ], + [ + "table", + "pc_recurrence_match", + "pc_recurrence_match", + "CREATE TABLE pc_recurrence_match (\n\tscope_id VARCHAR(256) NOT NULL, \n\tmatch_key VARCHAR(71) NOT NULL, \n\ttask_outcome_source_type VARCHAR(128) NOT NULL, \n\ttask_outcome_source_id VARCHAR(256) NOT NULL, \n\ttask_outcome_position BIGINT NOT NULL, \n\tfailure_item_kind VARCHAR(16) NOT NULL, \n\tfailure_item_index INTEGER NOT NULL, \n\tfailure_item_digest VARCHAR(71) NOT NULL, \n\tcandidate_set_mode VARCHAR(32) NOT NULL, \n\tcandidate_set_digest VARCHAR(71) NOT NULL, \n\tresult VARCHAR(16) NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\tsignature_key TEXT, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, match_key), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_recurrence_match_identity UNIQUE (scope_id, task_outcome_source_type, task_outcome_source_id, failure_item_kind, failure_item_index), \n\tCONSTRAINT ck_pc_recurrence_match_result CHECK (result IN ('matched', 'unmatched', 'ambiguous')), \n\tCONSTRAINT ck_pc_recurrence_match_candidate_set_mode CHECK (candidate_set_mode IN ('handoff_citations', 'scope_heads')), \n\tCONSTRAINT ck_pc_recurrence_match_failure_item_kind CHECK (failure_item_kind IN ('observation', 'check')), \n\tCONSTRAINT ck_pc_recurrence_match_position_positive CHECK (task_outcome_position > 0), \n\tCONSTRAINT ck_pc_recurrence_match_item_index_nonnegative CHECK (failure_item_index >= 0)\n)" + ], + [ + "table", + "pc_recurrence_observation", + "pc_recurrence_observation", + "CREATE TABLE pc_recurrence_observation (\n\tscope_id VARCHAR(256) NOT NULL, \n\tobservation_id VARCHAR(71) NOT NULL, \n\tselection_key VARCHAR(71) NOT NULL, \n\tverdict_key VARCHAR(71) NOT NULL, \n\tevent VARCHAR(16) NOT NULL, \n\tmatch_basis VARCHAR(8) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsignature_key TEXT NOT NULL, \n\tsignature_key_hash BLOB NOT NULL, \n\ttask_outcome_source_type VARCHAR(128) NOT NULL, \n\ttask_outcome_source_id VARCHAR(256) NOT NULL, \n\ttask_outcome_position BIGINT NOT NULL, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, observation_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_recurrence_observation_selection UNIQUE (scope_id, selection_key), \n\tCONSTRAINT uq_pc_recurrence_observation_verdict UNIQUE (scope_id, verdict_key), \n\tCONSTRAINT ck_pc_recurrence_observation_event CHECK (event IN ('selected', 'recurred', 'avoided')), \n\tCONSTRAINT ck_pc_recurrence_observation_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_recurrence_observation_position_positive CHECK (task_outcome_position > 0)\n)" + ], + [ + "table", + "pc_scope_bindings", + "pc_scope_bindings", + "CREATE TABLE pc_scope_bindings (\n\tintegration VARCHAR(128) NOT NULL, \n\tkind VARCHAR(64) NOT NULL, \n\texternal_id VARCHAR(256) NOT NULL, \n\texternal_id_search TEXT NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (integration, kind, external_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_scope_context_references", + "pc_scope_context_references", + "CREATE TABLE pc_scope_context_references (\n\tscope_id VARCHAR(256) NOT NULL, \n\treferenced_scope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (scope_id, referenced_scope_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(referenced_scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_scope_context_references_not_self CHECK (scope_id <> referenced_scope_id)\n)" + ], + [ + "table", + "pc_scope_creation_requests", + "pc_scope_creation_requests", + "CREATE TABLE pc_scope_creation_requests (\n\tidempotency_key VARCHAR(256) NOT NULL, \n\trequest_digest VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (idempotency_key), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_scope_external_references", + "pc_scope_external_references", + "CREATE TABLE pc_scope_external_references (\n\tscope_id VARCHAR(256) NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tkind VARCHAR(128) NOT NULL, \n\tvalue VARCHAR(2000) NOT NULL, \n\tvalue_search TEXT NOT NULL, \n\tvalue_digest VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, ordinal), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_scope_external_references_value UNIQUE (scope_id, kind, value_digest), \n\tCONSTRAINT ck_pc_scope_external_references_ordinal_nonnegative CHECK (ordinal >= 0)\n)" + ], + [ + "table", + "pc_scope_settings", + "pc_scope_settings", + "CREATE TABLE pc_scope_settings (\n\tname VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (name), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_scopes", + "pc_scopes", + "CREATE TABLE pc_scopes (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttitle VARCHAR(256) NOT NULL, \n\tsummary VARCHAR(2000) NOT NULL, \n\tscope_id_search TEXT NOT NULL, \n\ttitle_search TEXT NOT NULL, \n\tsummary_search TEXT NOT NULL, \n\tparent_scope_id VARCHAR(256), \n\tversion INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tFOREIGN KEY(parent_scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_scopes_version_positive CHECK (version > 0)\n)" + ], + [ + "table", + "pc_server_identity", + "pc_server_identity", + "CREATE TABLE pc_server_identity (\n\tsingleton_key INTEGER NOT NULL, \n\tserver_id VARCHAR(36) NOT NULL, \n\tPRIMARY KEY (singleton_key), \n\tCONSTRAINT ck_pc_server_identity_singleton CHECK (singleton_key = 1), \n\tUNIQUE (server_id)\n)" + ], + [ + "table", + "pc_skill_packages", + "pc_skill_packages", + "CREATE TABLE pc_skill_packages (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttree_digest VARCHAR(64) NOT NULL, \n\tarchive_digest VARCHAR(64) NOT NULL, \n\tarchive_bytes BLOB NOT NULL, \n\tmanifest BLOB NOT NULL, \n\tfile_count INTEGER NOT NULL, \n\tuncompressed_size BIGINT NOT NULL, \n\tarchive_size BIGINT NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, tree_digest), \n\tCONSTRAINT ck_pc_skill_packages_file_count CHECK (file_count > 0 AND file_count <= 256), \n\tCONSTRAINT ck_pc_skill_packages_uncompressed_size CHECK (uncompressed_size > 0 AND uncompressed_size <= 4194304), \n\tCONSTRAINT ck_pc_skill_packages_archive_size CHECK (archive_size > 0 AND archive_size <= 5242880)\n)" + ], + [ + "table", + "pc_skill_publications", + "pc_skill_publications", + "CREATE TABLE pc_skill_publications (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_id VARCHAR(64) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\tdesired_state VARCHAR(16) DEFAULT 'published' NOT NULL, \n\tdesired_revision INTEGER NOT NULL, \n\tdesired_tree_digest VARCHAR(64) NOT NULL, \n\tobserved_revision INTEGER, \n\tobserved_tree_digest VARCHAR(64), \n\tobserved_generation BIGINT, \n\tdestination TEXT, \n\tstate VARCHAR(32) NOT NULL, \n\tselected_runtime_variant VARCHAR(128), \n\tenvironment_fingerprint VARCHAR(64), \n\tlast_error_code VARCHAR(128), \n\tobserved_at DATETIME, \n\tgeneration BIGINT NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, target_id, artifact_id), \n\tCONSTRAINT ck_pc_skill_publications_desired_revision_positive CHECK (desired_revision > 0), \n\tCONSTRAINT ck_pc_skill_publications_observed_revision_positive CHECK (observed_revision IS NULL OR observed_revision > 0), \n\tCONSTRAINT ck_pc_skill_publications_desired_state CHECK (desired_state IN ('published', 'unpublished')), \n\tCONSTRAINT ck_pc_skill_publications_state CHECK (state IN ('unpublished', 'pending', 'current', 'update_available', 'delivery_failed', 'conflict', 'drifted', 'incompatible')), \n\tCONSTRAINT ck_pc_skill_publications_observed_generation_nonnegative CHECK (observed_generation IS NULL OR observed_generation >= 0), \n\tCONSTRAINT ck_pc_skill_publications_generation_nonnegative CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_source_cursors", + "pc_source_cursors", + "CREATE TABLE pc_source_cursors (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tcursor BLOB NOT NULL, \n\tgeneration BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, binding_name), \n\tCONSTRAINT ck_pc_source_cursors_generation_nonnegative CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_source_definition_manifests", + "pc_source_definition_manifests", + "CREATE TABLE pc_source_definition_manifests (\n\tdefinition_name VARCHAR(128) NOT NULL, \n\tdefinition_version VARCHAR(128) NOT NULL, \n\tfingerprint VARCHAR(71) NOT NULL, \n\tmanifest BLOB NOT NULL, \n\tPRIMARY KEY (definition_name, definition_version), \n\tCONSTRAINT uq_pc_source_definition_manifest_fingerprint UNIQUE (definition_name, fingerprint)\n)" + ], + [ + "table", + "pc_source_journal_heads", + "pc_source_journal_heads", + "CREATE TABLE pc_source_journal_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tposition BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tCONSTRAINT ck_pc_source_journal_heads_position_nonnegative CHECK (position >= 0)\n)" + ], + [ + "table", + "pc_source_write_plugins", + "pc_source_write_plugins", + "CREATE TABLE pc_source_write_plugins (\n\tplugin_id VARCHAR(128) NOT NULL, \n\tdefinition BLOB NOT NULL, \n\tPRIMARY KEY (plugin_id)\n)" + ], + [ + "table", + "pc_sources", + "pc_sources", + "CREATE TABLE pc_sources (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tpayload BLOB NOT NULL, \n\tjournal_position BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id), \n\tCONSTRAINT uq_pc_sources_scope_journal_position UNIQUE (scope_id, journal_position)\n)" + ], + [ + "table", + "pc_topic_memory_active_chunks", + "pc_topic_memory_active_chunks", + "CREATE TABLE pc_topic_memory_active_chunks (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tchunk_ordinal INTEGER NOT NULL, \n\tstart_offset INTEGER NOT NULL, \n\tend_offset INTEGER NOT NULL, \n\tchunk_text TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tpolicy_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, chunk_ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_topic_memory_revision_publications (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_ordinal_nonnegative CHECK (chunk_ordinal >= 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_start_nonnegative CHECK (start_offset >= 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_offsets CHECK (end_offset > start_offset)\n)" + ], + [ + "table", + "pc_topic_memory_active_topics", + "pc_topic_memory_active_topics", + "CREATE TABLE pc_topic_memory_active_topics (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\ttitle TEXT NOT NULL, \n\tsummary TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tsource_count INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_topic_memory_revision_publications (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_active_topics_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_active_topics_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_topic_memory_active_topics_sources_nonnegative CHECK (source_count >= 0)\n)" + ], + [ + "table", + "pc_topic_memory_chunk_fts", + "pc_topic_memory_chunk_fts", + "CREATE VIRTUAL TABLE pc_topic_memory_chunk_fts USING fts5(\n scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED,\n chunk_ordinal UNINDEXED, start_offset UNINDEXED,\n title UNINDEXED, summary UNINDEXED, chunk_text UNINDEXED,\n searchable_text, tokenize='unicode61'\n)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_config", + "pc_topic_memory_chunk_fts_config", + "CREATE TABLE 'pc_topic_memory_chunk_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_chunk_fts_content", + "pc_topic_memory_chunk_fts_content", + "CREATE TABLE 'pc_topic_memory_chunk_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4, c5, c6, c7, c8)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_data", + "pc_topic_memory_chunk_fts_data", + "CREATE TABLE 'pc_topic_memory_chunk_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_docsize", + "pc_topic_memory_chunk_fts_docsize", + "CREATE TABLE 'pc_topic_memory_chunk_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_idx", + "pc_topic_memory_chunk_fts_idx", + "CREATE TABLE 'pc_topic_memory_chunk_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_fts_index", + "pc_topic_memory_fts_index", + "CREATE TABLE pc_topic_memory_fts_index (\n\tsingleton INTEGER NOT NULL, \n\tschema_version INTEGER NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_topic_memory_fts_index_singleton CHECK (singleton = 1)\n)" + ], + [ + "table", + "pc_topic_memory_processing_targets", + "pc_topic_memory_processing_targets", + "CREATE TABLE pc_topic_memory_processing_targets (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_request_generation BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tcaptured_flush_generation BIGINT NOT NULL, \n\tobserved_dirty_generation BIGINT NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_topic_processing_target CHECK (target_request_generation > 0 AND source_through >= 0 AND captured_flush_generation >= 0 AND observed_dirty_generation >= 0)\n)" + ], + [ + "table", + "pc_topic_memory_retrieval_shape", + "pc_topic_memory_retrieval_shape", + "CREATE TABLE pc_topic_memory_retrieval_shape (\n\tsingleton INTEGER NOT NULL, \n\tshape VARCHAR(16) NOT NULL, \n\tprofile_fingerprint VARCHAR(64), \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_singleton CHECK (singleton = 1), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_value CHECK (shape IN ('fts', 'hybrid')), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_profile CHECK ((shape = 'fts' AND profile_fingerprint IS NULL) OR (shape = 'hybrid' AND profile_fingerprint IS NOT NULL))\n)" + ], + [ + "table", + "pc_topic_memory_revision_publications", + "pc_topic_memory_revision_publications", + "CREATE TABLE pc_topic_memory_revision_publications (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tpublished_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_publications_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_publications_revision_positive CHECK (revision > 0)\n)" + ], + [ + "table", + "pc_topic_memory_topic_fts", + "pc_topic_memory_topic_fts", + "CREATE VIRTUAL TABLE pc_topic_memory_topic_fts USING fts5(\n scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED,\n title UNINDEXED, summary UNINDEXED, searchable_text, tokenize='unicode61'\n)" + ], + [ + "table", + "pc_topic_memory_topic_fts_config", + "pc_topic_memory_topic_fts_config", + "CREATE TABLE 'pc_topic_memory_topic_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_topic_fts_content", + "pc_topic_memory_topic_fts_content", + "CREATE TABLE 'pc_topic_memory_topic_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4, c5)" + ], + [ + "table", + "pc_topic_memory_topic_fts_data", + "pc_topic_memory_topic_fts_data", + "CREATE TABLE 'pc_topic_memory_topic_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_topic_memory_topic_fts_docsize", + "pc_topic_memory_topic_fts_docsize", + "CREATE TABLE 'pc_topic_memory_topic_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_topic_memory_topic_fts_idx", + "pc_topic_memory_topic_fts_idx", + "CREATE TABLE 'pc_topic_memory_topic_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_work_blocks", + "pc_topic_memory_work_blocks", + "CREATE TABLE pc_topic_memory_work_blocks (\n\tblock_id VARCHAR(36) NOT NULL, \n\tattempt_id VARCHAR(36) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tsource_after BIGINT NOT NULL, \n\tsource_through BIGINT, \n\tbudget_source_through BIGINT, \n\tlegacy_source_through BIGINT, \n\ttarget_source_through BIGINT NOT NULL, \n\tattempts INTEGER NOT NULL, \n\trequests BIGINT NOT NULL, \n\ttokens BIGINT NOT NULL, \n\tlast_error_code VARCHAR(64), \n\terror_code VARCHAR(64) NOT NULL, \n\tblocked_at DATETIME NOT NULL, \n\tretried_at DATETIME, \n\tretry_request_generation BIGINT, \n\tPRIMARY KEY (block_id), \n\tCONSTRAINT ck_pc_topic_block_window CHECK (source_after >= 0 AND target_source_through > source_after AND (source_through IS NULL OR (source_through > source_after AND source_through <= target_source_through)) AND (budget_source_through IS NULL OR (budget_source_through > source_after AND budget_source_through <= target_source_through)) AND (source_through IS NULL OR budget_source_through IS NULL OR source_through <= budget_source_through) AND (legacy_source_through IS NULL OR (legacy_source_through > source_after AND legacy_source_through <= target_source_through))), \n\tCONSTRAINT ck_pc_topic_block_usage CHECK (attempts > 0 AND requests >= 0 AND tokens >= 0), \n\tCONSTRAINT ck_pc_topic_block_acceptance CHECK ((retried_at IS NULL AND retry_request_generation IS NULL) OR (retried_at IS NOT NULL AND retry_request_generation IS NOT NULL AND retry_request_generation > 0)), \n\tCONSTRAINT uq_pc_topic_block_identity UNIQUE (scope_id, binding_name, source_after, attempt_id)\n)" + ], + [ + "table", + "pc_topic_memory_work_budgets", + "pc_topic_memory_work_budgets", + "CREATE TABLE \"pc_topic_memory_work_budgets\" (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tsource_after BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tattempt_id VARCHAR(36) NOT NULL, \n\tattempts INTEGER NOT NULL, \n\trequests BIGINT NOT NULL, \n\ttokens BIGINT NOT NULL, \n\tfailure_code VARCHAR(64) NOT NULL, \n\tattempt_source_through BIGINT, \n\tstarted_at DATETIME, \n\tfinished_at DATETIME, \n\tsupervisor_group VARCHAR(64), \n\tsupervisor_generation BIGINT, \n\tholder_id VARCHAR(36), \n\tlast_error_code VARCHAR(64), \n\tPRIMARY KEY (scope_id, binding_name, source_after), \n\tCONSTRAINT ck_pc_topic_budget_window CHECK (source_after >= 0 AND source_through > source_after), \n\tCONSTRAINT ck_pc_topic_budget_usage CHECK (attempts > 0 AND requests >= 0 AND tokens >= 0), \n\tCONSTRAINT ck_pc_topic_budget_attempt_range CHECK (attempt_source_through IS NULL OR (attempt_source_through > source_after AND attempt_source_through <= source_through))\n)" + ], + [ + "trigger", + "pc_artifact_search_fts_delete", + "pc_artifact_search_text", + "CREATE TRIGGER pc_artifact_search_fts_delete AFTER delete ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id; END" + ], + [ + "trigger", + "pc_artifact_search_fts_insert", + "pc_artifact_search_text", + "CREATE TRIGGER pc_artifact_search_fts_insert AFTER insert ON pc_artifact_search_text BEGIN INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END" + ], + [ + "trigger", + "pc_artifact_search_fts_update", + "pc_artifact_search_text", + "CREATE TRIGGER pc_artifact_search_fts_update AFTER update ON pc_artifact_search_text BEGIN DELETE FROM pc_artifact_search_fts WHERE text_id = old.text_id;INSERT INTO pc_artifact_search_fts (text_id, scope_id, family, artifact_id, revision, build_id, index_def_id, searchable_text) VALUES (new.text_id, new.scope_id, new.family, new.artifact_id, new.revision, new.build_id, new.index_def_id, new.searchable_text); END" + ], + [ + "trigger", + "pc_atomic_memory_fts_delete", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_fts_delete AFTER DELETE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_fts WHERE identity_token MATCH ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); END" + ], + [ + "trigger", + "pc_atomic_memory_fts_insert", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_fts_insert AFTER INSERT ON pc_atomic_memory_current BEGIN INSERT INTO pc_atomic_memory_current_fts (scope_id, artifact_id, revision, state_version, content_hash, projection_format, kind, text, searchable_text, tag_keys, identity_token) VALUES (new.scope_id, new.artifact_id, new.revision, new.state_version, new.content_hash, new.projection_format, new.kind, new.text, new.searchable_text, new.tag_keys, 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id))); END" + ], + [ + "trigger", + "pc_atomic_memory_fts_update", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_fts_update AFTER UPDATE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_fts WHERE identity_token MATCH ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); INSERT INTO pc_atomic_memory_current_fts (scope_id, artifact_id, revision, state_version, content_hash, projection_format, kind, text, searchable_text, tag_keys, identity_token) VALUES (new.scope_id, new.artifact_id, new.revision, new.state_version, new.content_hash, new.projection_format, new.kind, new.text, new.searchable_text, new.tag_keys, 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id))); END" + ] + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_001_artifact_storage.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_001_artifact_storage.py new file mode 100644 index 0000000000..a8dd25e458 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_001_artifact_storage.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Register fixed Artifact plugin storage without mutating historical layouts.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "v1_5_0_001" +down_revision = "v1_4_1_006" + + +def upgrade(): + resource = Path(__file__).parents[1] / "schemas/sqlite-v1_5_0_001.json" + for statement in json.loads(resource.read_text())["statements"]: + op.get_bind().exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json index 66050c9009..0940df10b5 100644 --- a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json @@ -28,6 +28,10 @@ "schemas/sqlite-v1_4_1_005.json", "schemas/mysql-v1_4_1_005.json" ], + "v1_5_0_001": [ + "schemas/sqlite-v1_5_0_001.json", + "schemas/mysql-v1_5_0_001.json" + ], "v1_4_1_006": [ "schemas/sqlite-v1_4_1_006.json", "schemas/mysql-v1_4_1_006.json" @@ -55,6 +59,9 @@ "v1_4_1_005": [ "2e24be9f699c202cb9d7985690b2be947bdc98f8f64c1c4e4b8e16e35a580fc4" ], + "v1_5_0_001": [ + "8f2d4a32f3558612b1bd390c262b8b3e355b172e34cc99692f38c11876226ebd" + ], "v1_4_1_006": [ "cacce80bdfb14aff889b7babebc9ab8487b7f0a11d2dbbd3d77a3717a23df74c" ] @@ -62,12 +69,14 @@ "affected_objects": [ "pc_access_audit", "pc_access_idempotency", + "pc_access_owner_recovery", "pc_access_owners", "pc_access_relationship_heads", "pc_access_relationships", "pc_agent_skill_targets", "pc_artifact_candidate_heads", "pc_artifact_candidate_versions", + "pc_artifact_family_schemas", "pc_artifact_fts", "pc_artifact_fts_config", "pc_artifact_fts_content", @@ -76,6 +85,8 @@ "pc_artifact_fts_idx", "pc_artifact_handles", "pc_artifact_heads", + "pc_artifact_index_changes", + "pc_artifact_index_coverage", "pc_artifact_lineage_artifacts", "pc_artifact_lineage_sources", "pc_artifact_processing_auto_wave_targets", @@ -87,7 +98,21 @@ "pc_artifact_processing_schema", "pc_artifact_processing_sequences", "pc_artifact_publications", + "pc_artifact_search_fts", + "pc_artifact_search_fts_config", + "pc_artifact_search_fts_content", + "pc_artifact_search_fts_data", + "pc_artifact_search_fts_delete", + "pc_artifact_search_fts_docsize", + "pc_artifact_search_fts_idx", + "pc_artifact_search_fts_insert", + "pc_artifact_search_fts_update", + "pc_artifact_search_states", + "pc_artifact_search_text", "pc_artifact_tags", + "pc_artifact_vector_indexes", + "pc_artifact_vector_spaces", + "pc_artifact_vectors", "pc_artifacts", "pc_atomic_memory_current", "pc_atomic_memory_current_fts", @@ -104,16 +129,20 @@ "pc_dream_runs", "pc_external_skill_registrations", "pc_lexical_projection_versions", + "pc_memory_artifact_archive", "pc_memory_entry_heads", "pc_memory_entry_versions", "pc_memory_source_windows", "pc_model_usage_daily", + "pc_portable_restore_receipts", "pc_profile_policies", "pc_recall_effort_daily", "pc_recall_token_daily", "pc_receipt_migration_review", "pc_recurrence_match", "pc_recurrence_observation", + "pc_retained_dream_candidate_heads", + "pc_retained_dream_candidate_versions", "pc_scope_bindings", "pc_scope_context_references", "pc_scope_creation_requests", @@ -147,11 +176,6 @@ "pc_topic_memory_topic_fts_idx", "pc_topic_memory_work_blocks", "pc_topic_memory_work_budgets", - "pc_retained_dream_candidate_heads", - "pc_retained_dream_candidate_versions", - "pc_memory_artifact_archive", - "pc_portable_restore_receipts", - "pc_access_owner_recovery", "pc_source_write_plugins" ], "readiness_scope": "server", @@ -198,6 +222,27 @@ "seekdb", "oceanbase" ] + }, + { + "id": "artifact_storage", + "revision": "v1_5_0_001", + "kind": "data", + "module": "artifact_storage_readiness.py", + "resources": [ + "server_domain.py", + "server_vector_domain.py", + "atomic_memory_readiness_1300.py", + "dream/upgrade.py", + "dream/schema.json" + ], + "depends_on": [ + "portable_storage" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] } ], "domain_readiness": { @@ -219,6 +264,10 @@ "server_upgrade", "portable_storage" ], + "v1_5_0_001": [ + "artifact_storage", + "portable_storage" + ], "v1_4_1_006": [ "server_upgrade", "portable_storage" @@ -239,6 +288,7 @@ "schemas/mysql-s1402.json", "schemas/mysql-v1_4_1_004.json", "schemas/mysql-v1_4_1_005.json", + "schemas/mysql-v1_5_0_001.json", "schemas/mysql-v1_4_1_006.json" ], "mysql_history": "mysql", diff --git a/src/powercontext/builtin/persistence/migrations/vector_profile.py b/src/powercontext/builtin/persistence/migrations/vector_profile.py index cd1d54dc87..bb476f703f 100644 --- a/src/powercontext/builtin/persistence/migrations/vector_profile.py +++ b/src/powercontext/builtin/persistence/migrations/vector_profile.py @@ -60,7 +60,7 @@ def __init__(self, directory: Path, profile: EmbeddingProfile): "profile": profile.model_dump(mode="json"), }) - def sqlite_layouts(self, directory: Path) -> dict[str, list[list[Any]]]: + def sqlite_layouts(self, directory: Path) -> dict[str, list[list[Any]]]: # noqa: C901 - Compose the registered revision layouts in chain order. core = json.loads((directory / "schemas/sqlite.json").read_text(encoding="utf-8")) archive = json.loads((directory / "schemas/sqlite-s1401.json").read_text(encoding="utf-8"))["create"] layouts = { @@ -106,6 +106,15 @@ def sqlite_layouts(self, directory: Path) -> dict[str, list[list[Any]]]: layouts["v1_4_1_006"] = layouts["v1_4_1_005"] + [ ["table", "pc_source_write_plugins", "pc_source_write_plugins", plugin] ] + artifact_revision = "v1_5_0_001" + if artifact_revision in manifest["revision_resources"]: + artifact = json.loads((directory / f"schemas/sqlite-{artifact_revision}.json").read_text())["layout"] + previous = {(row[0], row[1]): row for row in layouts["v1_4_1_006"]} + for row in artifact: + key = (row[0], row[1]) + if key not in previous or row[1] == "pc_artifact_processing_binding_states": + previous[key] = row + layouts[artifact_revision] = list(previous.values()) for entries in layouts.values(): entries.sort(key=lambda row: (row[0], row[1])) return layouts diff --git a/src/powercontext/builtin/persistence/processing_intents.py b/src/powercontext/builtin/persistence/processing_intents.py index 81174a5600..f406d8bab4 100644 --- a/src/powercontext/builtin/persistence/processing_intents.py +++ b/src/powercontext/builtin/persistence/processing_intents.py @@ -339,7 +339,13 @@ async def admit( state = ( ( await connection.execute( - select(state_table).where(state_table.c.binding_name == binding_name).with_for_update() + select( + state_table.c.scan_in_progress, + state_table.c.scan_generation, + state_table.c.scan_upper_pending_sequence, + ) + .where(state_table.c.binding_name == binding_name) + .with_for_update() ) ) .mappings() diff --git a/src/powercontext/builtin/persistence/processing_migration.py b/src/powercontext/builtin/persistence/processing_migration.py index 15e5fad831..9c17544769 100644 --- a/src/powercontext/builtin/persistence/processing_migration.py +++ b/src/powercontext/builtin/persistence/processing_migration.py @@ -297,6 +297,9 @@ async def apply_processing_migration( # noqa: C901 - explicit durable maintenan # Names/definitions are fixed constants, never user SQL. await connection.exec_driver_sql(f"ALTER TABLE {STATES.name} ADD COLUMN {name} {definition}") return ProcessingMigrationProgress(phase="schema") + from powercontext.builtin.persistence.custom_artifacts import ensure_artifact_plugin_schema + + await ensure_artifact_plugin_schema(connection) problems = await _schema_issues(connection) if problems: raise ProcessingSchemaNotReadyError(reason="; ".join(problems)) diff --git a/src/powercontext/builtin/persistence/records.py b/src/powercontext/builtin/persistence/records.py index 42f249acd2..4c06951b4b 100644 --- a/src/powercontext/builtin/persistence/records.py +++ b/src/powercontext/builtin/persistence/records.py @@ -33,6 +33,7 @@ from powercontext.builtin.persistence.artifact_handles import ArtifactHandleRepository from powercontext.builtin.persistence.artifacts import ArtifactRepository, with_artifact_content from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import ( RepositoryNotFoundError, @@ -59,6 +60,7 @@ ) from powercontext.builtin.persistence.sources import SourceRepository, StoredSource from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE, SOURCE_DEFINITION_MANIFESTS_TABLE, @@ -476,12 +478,12 @@ async def create_atomic_memories( async def get_artifact( self, scope_id: str, family: str, artifact_id: str, /, *, execution_context=None ) -> ArtifactRecord: - self._require_family(family) + repository = await self._read_repository(family) if family == "atomic-memory" and execution_context is not None: return await self._get_authorized_atomic_artifact(scope_id, artifact_id, None, execution_context) async with self._database.transaction() as connection: try: - artifact = await self._artifacts.latest(connection, scope_id, family, artifact_id) + artifact = await repository.latest(connection, scope_id, family, artifact_id) except RepositoryNotFoundError: raise BaseValueNotFoundError("artifact", (scope_id, family, artifact_id)) from None return _artifact_record(scope_id, artifact) @@ -502,12 +504,12 @@ async def get_artifact_revision( *, execution_context=None, ) -> ArtifactRecord: - self._require_family(family) + repository = await self._read_repository(family) if family == "atomic-memory" and execution_context is not None: return await self._get_authorized_atomic_artifact(scope_id, artifact_id, revision, execution_context) async with self._database.transaction() as connection: try: - artifact = await self._artifacts.get( + artifact = await repository.get( connection, scope_id, ArtifactRef(family=family, artifact_id=artifact_id, revision=revision), @@ -545,13 +547,14 @@ async def list_artifact_revisions( rows, _, next_cursor = await self.revision_rows( connection, scope_id, family, artifact_id, limit=limit, cursor=cursor ) - artifacts = await self._artifacts.decode_many(connection, scope_id, rows) + repository = await self._read_repository(family, connection=connection) + artifacts = await repository.decode_many(connection, scope_id, rows) items = tuple(_artifact_collection_item(scope_id, artifact) for artifact in artifacts) return ArtifactRevisionPage(items=items, next_cursor=next_cursor) async def revision_rows(self, connection, scope_id, family, artifact_id, *, limit, cursor, columns=()): """Read a bounded immutable revision page through a borrowed transaction.""" - self._require_family(family) + await self._read_repository(family, connection=connection) _require_limit(limit) expected_cursor = { "version": 1, @@ -631,7 +634,7 @@ async def query_artifacts( cursor: str | None, tag_filter: TagFilter | None = None, ) -> ArtifactRecordPage: - self._require_family(family) + repository = await self._read_repository(family) _require_limit(limit) reader = self._topic_memory_list_reader if reader is not None and family == reader.family: @@ -693,7 +696,7 @@ async def query_artifacts( .all() ) selected_rows = rows[:limit] - artifacts = await self._artifacts.decode_many(connection, scope_id, selected_rows) + artifacts = await repository.decode_many(connection, scope_id, selected_rows) items = tuple(_artifact_collection_item(scope_id, artifact) for artifact in artifacts) next_cursor = None @@ -834,11 +837,22 @@ def _require_content_source(self, source_type: str) -> None: if source_type != CONTENT_SOURCE_NAME: raise InvalidBaseAccessRequestError("source_type", "must be content") - def _require_family(self, family: str) -> None: + async def _read_repository(self, family: str, *, connection: AsyncConnection | None = None) -> ArtifactRepository: if family == "memory": raise BaseOperationNotSupportedError("artifact_family", family, "legacy Memory collection read") - if family not in self._artifacts.families: + if family in self._artifacts.families: + return self._artifacts + if connection is None: + async with self._database.transaction() as borrowed: + return await self._read_repository(family, connection=borrowed) + known = await connection.scalar( + select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family) + .where(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family == family) + .limit(1) + ) + if known is None: raise InvalidBaseAccessRequestError("family", "must be a registered Artifact family") + return custom_artifact_repository(family) async def _get_source( self, @@ -917,9 +931,14 @@ def _artifact_record( artifact: Artifact[Any], ) -> ArtifactRecord: content = cast(dict[str, JsonValue], artifact.content.model_dump(mode="json", by_alias=True)) + schema_version = None + if isinstance(artifact.content, StoredArtifactContent): + schema_version = artifact.content.schema_version + content = artifact.content.content return ArtifactRecord( scope_id=scope_id, family=cast(Any, artifact.family), + schema_version=schema_version, artifact_id=artifact.artifact_id, revision=artifact.revision, content=content, @@ -943,9 +962,14 @@ def _artifact_created(scope_id: str, artifact: Artifact[Any]) -> ArtifactCreated def _artifact_collection_item(scope_id: str, artifact: Artifact[Any]) -> ArtifactCollectionItem: content = cast(dict[str, JsonValue], artifact.content.model_dump(mode="json", by_alias=True)) + schema_version = None + if isinstance(artifact.content, StoredArtifactContent): + schema_version = artifact.content.schema_version + content = artifact.content.content return ArtifactCollectionItem( scope_id=scope_id, family=cast(Any, artifact.family), + schema_version=schema_version, artifact_id=artifact.artifact_id, revision=artifact.revision, sources=artifact.lineage.sources, diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index 26b6dc4d11..b82a72dc52 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -26,9 +26,14 @@ from powercontext.builtin.persistence.bootstrap import mysql_error_code -async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: +async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: # noqa: C901 - Keep unmanaged backend creation and its derived structures together. """Create selected tables only for unmanaged storage; managed schemas are read-only.""" + from powercontext.builtin.persistence.tables import ARTIFACT_VECTORS_TABLE + + tables = tuple( + table for table in tables if table is not ARTIFACT_VECTORS_TABLE or connection.dialect.name == "sqlite" + ) names = await connection.run_sync(lambda sync: set(inspect(sync).get_table_names())) if "pc_schema_revision" in names: if not {table.name for table in tables} <= names: @@ -40,20 +45,37 @@ async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) return if connection.dialect.name == "mysql": await connection.run_sync(lambda sync: _create_mysql_tables(sync, tables)) - return + else: + groups: dict[MetaData, list[Table]] = {} + for table in tables: + groups.setdefault(table.metadata, []).append(table) + + for metadata, selected in groups.items(): + await connection.run_sync( + lambda sync_connection, current=metadata, owned=selected: current.create_all( + sync_connection, + tables=owned, + checkfirst=True, + ) + ) - groups: dict[MetaData, list[Table]] = {} - for table in tables: - groups.setdefault(table.metadata, []).append(table) + if any(table.name == "pc_artifact_search_text" for table in tables): + from powercontext.builtin.persistence.artifact_search_schema import ensure_search_support - for metadata, selected in groups.items(): - await connection.run_sync( - lambda sync_connection, current=metadata, owned=selected: current.create_all( - sync_connection, - tables=owned, - checkfirst=True, - ) - ) + await connection.run_sync(ensure_search_support) + if connection.dialect.name != "sqlite" and any(table.name == "pc_artifact_vector_spaces" for table in tables): + from sqlalchemy import select + + from powercontext.builtin.persistence.artifact_indexes import SCHEMAS, register_definition, search_declaration + from powercontext.builtin.persistence.artifact_vector_space import provision_native_spaces + + rows = (await connection.execute(select(SCHEMAS))).mappings().all() + for row in rows: + declaration = search_declaration(row["declaration"]) + if declaration is not None: + for index in declaration.indexes: + await register_definition(connection, row["family"], row["schema_version"], index) + await connection.run_sync(provision_native_spaces) def _create_mysql_tables(connection: Connection, tables: Sequence[Table]) -> None: diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 4505f2b742..1488898501 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -40,7 +40,11 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository -from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE +from powercontext.builtin.persistence.tables import ( + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, +) from powercontext.builtin.sources.content import ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError @@ -198,9 +202,9 @@ async def add_many_with_status( .values(payload=payload) ) if new_rows: - await ArtifactProcessingIntentRepository().mark_dirty_many( - connection, scope_id, SOURCE_PROCESSING_BINDINGS, increment=len(new_rows) - ) + intents = ArtifactProcessingIntentRepository() + await intents.mark_dirty_many(connection, scope_id, SOURCE_PROCESSING_BINDINGS, increment=len(new_rows)) + await _mark_custom_bindings_dirty(connection, scope_id, intents, increment=len(new_rows)) return tuple(results) def _encode(self, source: Source) -> tuple[SourceRef, bytes]: @@ -573,3 +577,24 @@ def _require_identity(field: str, value: object, maximum: int) -> None: raise InvalidRepositoryArgumentError(field, "must be a non-empty trimmed string") if len(value) > maximum: raise InvalidRepositoryArgumentError(field, f"must not exceed {maximum} characters") + + +async def _mark_custom_bindings_dirty( + connection: AsyncConnection, + scope_id: str, + intents: ArtifactProcessingIntentRepository, + *, + increment: int, +) -> None: + plugins = ( + await connection.execute( + select( + ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.binding_name, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.plugin_spec[("manifest", "scope_ids")].label("scope_ids"), + ).where(ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.plugin_spec.is_not(None)) + ) + ).all() + bindings = tuple( + plugin.binding_name for plugin in plugins if plugin.scope_ids is None or scope_id in plugin.scope_ids + ) + await intents.mark_dirty_many(connection, scope_id, bindings, increment=increment) diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index 236c4a0d57..8ed39180b0 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -34,7 +34,7 @@ from sqlalchemy.engine import URL, AdaptedConnection, ExceptionContext, make_url from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.exc import OperationalError -from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine from sqlalchemy.pool import ConnectionPoolEntry, StaticPool from powercontext.builtin.persistence.database import AsyncDatabase @@ -208,6 +208,7 @@ def set_pragmas(dbapi_connection: DBAPIConnection, connection_record: Connection connection_record.info["_powercontext_sqlite_stop"] = _make_sqlite_stop_idempotent(driver) if load_vector_extension: dbapi_connection.run_async(_load_sqlite_vec) + connection_record.info["powercontext_sqlite_vec_loaded"] = True cursor = dbapi_connection.cursor() try: cursor.execute(f"PRAGMA busy_timeout = {config.busy_timeout_ms}") @@ -272,6 +273,14 @@ def stop_once() -> asyncio.Future[Any] | None: return stop_once +async def ensure_sqlite_vector_extension(connection: AsyncConnection) -> None: + """Load vector functions only when this connection first serves vector search.""" + if not connection.info.get("powercontext_sqlite_vec_loaded"): + raw = await connection.get_raw_connection() + await _load_sqlite_vec(cast(Connection, raw.driver_connection)) + connection.info["powercontext_sqlite_vec_loaded"] = True + + async def _load_sqlite_vec(connection: Connection) -> None: await connection.enable_load_extension(True) try: diff --git a/src/powercontext/builtin/persistence/supervision.py b/src/powercontext/builtin/persistence/supervision.py index fcc3eced25..021d673084 100644 --- a/src/powercontext/builtin/persistence/supervision.py +++ b/src/powercontext/builtin/persistence/supervision.py @@ -291,9 +291,15 @@ async def load( .where(ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.binding_name == binding_name) .values(scan_generation=ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.scan_generation) ) - statement = select(ARTIFACT_PROCESSING_BINDING_STATES_TABLE).where( - ARTIFACT_PROCESSING_BINDING_STATES_TABLE.c.binding_name == binding_name - ) + table = ARTIFACT_PROCESSING_BINDING_STATES_TABLE + statement = select( + table.c.binding_name, + table.c.last_auto_wave_completed_at, + table.c.last_schedule_checkpoint_at, + table.c.scan_generation, + table.c.scan_in_progress, + table.c.scan_upper_pending_sequence, + ).where(table.c.binding_name == binding_name) if for_update: statement = statement.with_for_update() row = (await connection.execute(statement)).mappings().one_or_none() diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 1b223bc8da..226e8d9ac1 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -15,6 +15,7 @@ """Shared dialect-neutral relational tables.""" from sqlalchemy import ( + JSON, BigInteger, Boolean, CheckConstraint, @@ -34,6 +35,7 @@ ) from sqlalchemy.dialects.mysql import BINARY, DATETIME, MEDIUMBLOB, MEDIUMTEXT, VARCHAR +from powercontext.builtin.persistence.artifact_search_schema import artifact_search_tables from powercontext.limits import ( MAX_ARTIFACT_FAMILY_LENGTH, MAX_ARTIFACT_HANDLE_KEY_LENGTH, @@ -568,9 +570,24 @@ def _entry_text_type(): Column("scan_generation", BigInteger, nullable=False, server_default="0"), Column("scan_in_progress", Boolean, nullable=False, server_default="0"), Column("scan_upper_pending_sequence", BigInteger), + Column("plugin_id", String(128)), + Column("plugin_family", String(128)), + Column("plugin_spec", JSON(none_as_null=True)), + Column("binding_generation", BigInteger, nullable=False, server_default="0"), + Column("desired_state", String(16)), + Column("phase", String(16)), + Column("last_error", String(1024)), CheckConstraint("scan_generation >= 0", name="ck_pc_processing_scan_generation"), ) +ARTIFACT_FAMILY_SCHEMAS_TABLE = Table( + "pc_artifact_family_schemas", + SHARED_METADATA, + Column("family", identity_string(128), primary_key=True), + Column("schema_version", identity_string(128), primary_key=True), + Column("declaration", JSON, nullable=False), +) + ARTIFACT_PROCESSING_SEQUENCES_TABLE = Table( "pc_artifact_processing_sequences", SHARED_METADATA, @@ -1065,6 +1082,7 @@ def _entry_text_type(): SHARED_TABLES = ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE, ARTIFACTS_TABLE, @@ -1430,3 +1448,17 @@ def _entry_text_type(): LEXICAL_PROJECTION_VERSIONS_TABLE, ) ) + + +# Search builds are independent of generation bindings. + +( + ARTIFACT_VECTOR_SPACES_TABLE, + ARTIFACT_VECTOR_INDEXES_TABLE, + ARTIFACT_SEARCH_STATES_TABLE, + ARTIFACT_INDEX_COVERAGE_TABLE, + ARTIFACT_INDEX_CHANGES_TABLE, + ARTIFACT_VECTORS_TABLE, + ARTIFACT_SEARCH_TEXT_TABLE, +) = ARTIFACT_SEARCH_TABLES = artifact_search_tables(SHARED_METADATA, identity_string) +BUILTIN_TABLES += ARTIFACT_SEARCH_TABLES diff --git a/src/powercontext/builtin/records.py b/src/powercontext/builtin/records.py index 4924442f08..9cb8106e37 100644 --- a/src/powercontext/builtin/records.py +++ b/src/powercontext/builtin/records.py @@ -31,7 +31,7 @@ from powercontext.builtin.tags import ArtifactTagSet, TagFilter, TagQuery, TagQueryPage, TagTarget BaseArtifactFamily = Literal["atomic-memory", "experience", "skill", "handoff", "profile", "prompt", "topic-memory"] -ArtifactReadFamily = Literal["atomic-memory", "experience", "skill", "handoff", "profile", "prompt", "topic-memory"] +ArtifactReadFamily = str class _RecordModel(BaseModel): @@ -89,6 +89,7 @@ class ArtifactRecord(_RecordModel): scope_id: str family: ArtifactReadFamily + schema_version: str | None = None artifact_id: str revision: int content: dict[str, JsonValue] @@ -106,6 +107,7 @@ class ArtifactCollectionItem(_RecordModel): scope_id: str family: ArtifactReadFamily + schema_version: str | None = None artifact_id: str revision: int sources: tuple[SourceRef, ...] diff --git a/src/powercontext/builtin/runtime/artifact_plugins.py b/src/powercontext/builtin/runtime/artifact_plugins.py new file mode 100644 index 0000000000..db91b2f59e --- /dev/null +++ b/src/powercontext/builtin/runtime/artifact_plugins.py @@ -0,0 +1,345 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Execute validated Artifact plugins in an owned generation Worker.""" + +from __future__ import annotations + +import asyncio +import json +import logging +from contextlib import AsyncExitStack +from dataclasses import dataclass +from typing import Any + +from jsonschema import Draft202012Validator +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext._logging import log_safely +from powercontext.artifact_plugins import ( + ArtifactDraft, + GenerationBatch, + GenerationContext, + GenerationSource, +) +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.artifact_plugin_validation import ( + _MAX_INPUT_BYTES, + _MAX_OUTPUT_BYTES, + _PluginManifest, + code_fingerprint, + describe_module, + load_plugin, +) +from powercontext.builtin.persistence.artifacts import RepositoryArtifactDraft +from powercontext.builtin.persistence.cursors import SourceCursorRepository +from powercontext.builtin.persistence.custom_artifacts import ( + StoredArtifactContent, + custom_artifact_repository, +) +from powercontext.builtin.persistence.generation_sources import GenerationSourceAccess +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, +) +from powercontext.builtin.runtime.artifact_reads import ArtifactReadSession +from powercontext.builtin.runtime.config import BuiltinConfig, DatabaseConfig, InferenceConfig, RuntimeConfig +from powercontext.builtin.runtime.processing_contracts import ( + ArtifactProcessingWorkAssignment, + ArtifactProcessingWorkerCompletion, +) +from powercontext.builtin.runtime.processing_execution import InvocationAlreadyHandled, ScopeInvocation +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, SourceCursor +from powercontext.sources import SourceProjectionKey + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ArtifactPluginWorkerSpec: + database: DatabaseConfig + binding_name: str + generation: int + definition: dict[str, Any] + security: dict[str, Any] | None = None + inference_config: Any = None + runtime_config: Any = None + + +async def require_current_binding(connection: AsyncConnection, spec: ArtifactPluginWorkerSpec, scope_id: str) -> None: + table = ARTIFACT_PROCESSING_BINDING_STATES_TABLE + locked = await connection.execute( + update(table) + .where( + table.c.binding_name == spec.binding_name, + table.c.binding_generation == spec.generation, + table.c.phase == "enabled", + ) + .values(binding_generation=table.c.binding_generation) + ) + scope_ids = spec.definition["manifest"]["scope_ids"] + if locked.rowcount != 1 or (scope_ids is not None and scope_id not in scope_ids): + raise ValueError("plugin invocation has been revoked") # noqa: TRY003 + + +def run_artifact_plugin_worker( + spec: ArtifactPluginWorkerSpec, + assignment: ArtifactProcessingWorkAssignment, +) -> ArtifactProcessingWorkerCompletion: + return asyncio.run(_process_window(spec, assignment)) + + +async def _restore_access_families(database) -> None: + from powercontext.server.authz.profiles import register_custom_artifact_access_family + + async with database.transaction() as connection: + families = (await connection.execute(select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family).distinct())).scalars().all() + for family in families: + register_custom_artifact_access_family(family) + + +async def _process_window(spec: ArtifactPluginWorkerSpec, assignment: ArtifactProcessingWorkAssignment): # noqa: C901 - One authorized window with preparation outside its commit. + # Server authorization is reconstructed in this child using deployment identity. + from powercontext.server.processing_security import open_worker_security + + definition = spec.definition + declaration = definition["declaration"] + if assignment.binding_name != spec.binding_name or assignment.artifact_family != declaration["family"]: + raise ValueError("assignment does not match the registered plugin") # noqa: TRY003 + sources = SourceRepository(BUILTIN_SOURCE_REGISTRY) + cursors = SourceCursorRepository() + async with AsyncExitStack() as resources: + from powercontext.builtin.persistence.artifact_indexes import ArtifactProjectionPublisher + from powercontext.builtin.runtime.composition import ( + _embedding_models, + compose_artifact_search, + open_builtin_contexts, + ) + + embedding = None + if spec.inference_config is not None: + embedding, _ = await _embedding_models(spec.inference_config, resources, None) + config = BuiltinConfig( + database=spec.database, + inference=spec.inference_config or InferenceConfig(), + runtime=spec.runtime_config or RuntimeConfig(), + ) + storage = await resources.enter_async_context( + open_builtin_contexts(config, embedding_model=embedding, _topic_memory_worker=True) + ) + search_service, _ = compose_artifact_search(storage, embedding) + publisher = ArtifactProjectionPublisher(storage.database, embedding) + await _restore_access_families(storage.database) + security = await resources.enter_async_context(open_worker_security(spec.security, storage.database)) + + async def authorize(connection): + await require_current_binding(connection, spec, assignment.scope_id) + if security is not None: + await security.authorize_transaction(connection, assignment.scope_id) + + invocation = ScopeInvocation(assignment, authorize_transaction=authorize) + try: + async with storage.database.transaction() as connection: + await invocation.start(connection) + cursor = await cursors.load(connection, assignment.scope_id, spec.binding_name) + after = 0 if cursor is None else cursor.cursor.sequence + high = await sources.journal_position(connection, assignment.scope_id) + through = min(high, after + definition["manifest"]["max_sources"]) + rows = await GenerationSourceAccess(sources).list_window_for_generation( + connection, + assignment.scope_id, + after=after, + through=through, + ) + # Desired index builds belong to maintenance. Generation prepares only the + # current serving projections, so an unavailable future index cannot block writes. + projections = { + name: SourceProjectionKey.model_validate(key) for name, key in declaration["projections"].items() + } + batch = GenerationBatch( + sources=tuple( + GenerationSource( + ref=row.ref, + projection=projections[row.ref.source_type], + value=BUILTIN_SOURCE_REGISTRY.project(row.value, projections[row.ref.source_type]), + ) + for row in rows + if row.ref.source_type in projections + ) + ) + if ( + len( + json.dumps( + [source.model_dump(mode="json") for source in batch.sources], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ) + > _MAX_INPUT_BYTES + ): + raise ValueError("input evidence exceeds the window budget") # noqa: TRY003 + source_evidence = frozenset((source.ref.source_type, source.ref.source_id) for source in batch.sources) + artifact_evidence = ArtifactReadSession( + storage.database, + assignment.scope_id, + {declaration["family"], *declaration.get("artifact_families", ())}, + access=None if security is None else security.access, + principal=None if security is None else security.principal, + audit=None if security is None else security.context, + guard=invocation.guard, + search_service=search_service, + ) + + drafts = [] + if batch.sources: + if code_fingerprint(definition["directory"]) != declaration["fingerprint"]: + raise ValueError("plugin files changed after registration") # noqa: TRY003 + manifest = _PluginManifest.model_validate(definition["manifest"]) + module = load_plugin(definition["directory"], manifest) + if describe_module(module, manifest, definition["directory"]) != { + "manifest": definition["manifest"], + "declaration": declaration, + }: + raise ValueError("Worker declaration differs from the registered definition") # noqa: TRY003 + ctx = GenerationContext( + assignment.scope_id, + resources, + artifact_evidence.read, + list_artifacts=artifact_evidence.list, + search_artifacts=artifact_evidence.search, + ) + pipeline = await module.create_pipeline( + ctx, module.Config.model_validate_json(json.dumps(manifest.config, ensure_ascii=False)) + ) + drafts = await pipeline.generate(batch) + validated = _validate_drafts(declaration, source_evidence, artifact_evidence.refs, drafts) + prepared = {} + for draft in validated: + prepared[draft.artifact_id] = await publisher.prepare( + assignment.scope_id, draft.family, draft.schema_version, draft.content + ) + await _commit_window( + storage.database, + spec, + assignment, + declaration, + invocation, + sources, + cursor, + through, + source_evidence, + artifact_evidence.refs, + drafts, + security, + prepared, + ) + except InvocationAlreadyHandled: + pass + return ArtifactProcessingWorkerCompletion() + + +def _validate_drafts(declaration, source_evidence, artifact_evidence, drafts) -> list[ArtifactDraft]: + if not isinstance(drafts, list): + raise TypeError("plugin must return a list of drafts") # noqa: TRY003 + validated = [ArtifactDraft.model_validate(draft) for draft in drafts] + if ( + len(json.dumps([draft.model_dump(mode="json") for draft in validated], ensure_ascii=False).encode()) + > _MAX_OUTPUT_BYTES + ): + raise ValueError("output exceeds the window budget") # noqa: TRY003 + validator = Draft202012Validator(declaration["content_schema"]) + identities = set() + for draft in validated: + if (draft.family, draft.schema_version) != (declaration["family"], declaration["schema_version"]): + raise ValueError("draft does not match the registered family/schema") # noqa: TRY003 + if draft.artifact_id in identities or not { + (ref.source_type, ref.source_id) for ref in draft.source_refs + }.issubset(source_evidence): + raise ValueError("duplicate artifact or Source evidence outside this invocation") # noqa: TRY003 + if not {(ref.family, ref.artifact_id, ref.revision) for ref in draft.artifact_refs}.issubset(artifact_evidence): + raise ValueError("Artifact evidence was not read through this invocation") # noqa: TRY003 + if ( + draft.expected_revision > 0 + and (draft.family, draft.artifact_id, draft.expected_revision) not in artifact_evidence + ): + raise ValueError("update revision must have been read through this invocation") # noqa: TRY003 + identities.add(draft.artifact_id) + validator.validate(draft.content) + return validated + + +async def _commit_window( + database, + spec, + assignment, + declaration, + invocation, + sources, + cursor, + through, + source_evidence, + artifact_evidence, + drafts, + security, + prepared_indexes=None, +): + from powercontext.server.authz import AccessAction, ResourceRef + + validated = _validate_drafts(declaration, source_evidence, artifact_evidence, drafts) + repository = custom_artifact_repository(declaration["family"], sources, prepared=prepared_indexes) + async with database.transaction() as connection: + # The Worker guard rechecks Scope contribution, which includes evidence reads. + await invocation.guard(connection) + for draft in validated: + resource = ResourceRef.artifact(assignment.scope_id, family=draft.family, artifact_id=draft.artifact_id) + bound = None if security is None else security.access.with_connection(connection) + prepared = RepositoryArtifactDraft( + family=draft.family, + content=StoredArtifactContent(schema_version=draft.schema_version, content=draft.content), + sources=draft.source_refs, + artifacts=draft.artifact_refs, + ) + if draft.expected_revision == 0: + await repository.create(connection, assignment.scope_id, draft.artifact_id, prepared) + if bound is not None: + await bound.establish_artifact_owner( + resource, + security.principal, + idempotency_key=f"background-custom-owner:{assignment.scope_id}:{draft.family}:{draft.artifact_id}", + context=security.context, + ) + else: + if bound is not None: + await bound.require( + security.principal, AccessAction.ARTIFACT_WRITE, resource, context=security.context + ) + previous = await repository.get( + connection, + assignment.scope_id, + ArtifactRef(family=draft.family, artifact_id=draft.artifact_id, revision=draft.expected_revision), + ) + await repository.revise(connection, assignment.scope_id, previous, prepared) + if through > (0 if cursor is None else cursor.cursor.sequence): + await SourceCursorRepository().save( + connection, + assignment.scope_id, + spec.binding_name, + SourceCursor(sequence=through), + expected_generation=None if cursor is None else cursor.generation, + ) + high = await sources.journal_position(connection, assignment.scope_id) + await invocation.complete(connection, remaining_work=high > through) + log_safely(logger, logging.DEBUG, "Artifact plugin window committed") diff --git a/src/powercontext/builtin/runtime/artifact_processing.py b/src/powercontext/builtin/runtime/artifact_processing.py index caf993238d..cb53382c23 100644 --- a/src/powercontext/builtin/runtime/artifact_processing.py +++ b/src/powercontext/builtin/runtime/artifact_processing.py @@ -133,6 +133,8 @@ class ArtifactProcessingBinding: # Applies to every invocation, including explicit requests and retries. # Domains inspect durable metadata here; they retain their Worker guards. work_block: Callable[[AsyncConnection, str, str], Awaitable[ArtifactProcessingBlock | None]] | None = None + # Hot-managed definitions can revoke a Scope even for an older accepted request. + invocation_scope_filter: Callable[[AsyncConnection, tuple[str, ...]], Awaitable[frozenset[str]]] | None = None # Receives one fixed telemetry summary after every worker execution attempt. # It must not affect domain scheduling or completion semantics. worker_model_usage_observer: Callable[[ArtifactProcessingModelUsageDelta | None], None] | None = None @@ -557,9 +559,14 @@ def __init__( self._rotation = 0 self._lease_lost = False self._unreaped: list[ArtifactProcessingWorkerHandle] = [] + self._binding_changes: asyncio.Queue[tuple[str, ArtifactProcessingBinding | None, asyncio.Future[None]]] = ( + asyncio.Queue() + ) @property def status(self) -> ArtifactProcessingSupervisorStatus: + if not self._families: + return ArtifactProcessingSupervisorStatus.DISABLED if self._fence is not None and any(state.degraded for state in self._families.values()): return ArtifactProcessingSupervisorStatus.DEGRADED return self._status @@ -651,6 +658,42 @@ def wake(self, binding_name: str | None = None) -> None: state.discovery_pending = True self._wake.set() + async def replace_binding(self, name: str, binding: ArtifactProcessingBinding | None) -> None: + """Change one global binding at a scheduling boundary and reap its old children.""" + + if self._supervisor_group != "global" or self._task is None or self._stop.is_set(): + raise ValueError("hot registration requires a running global Supervisor") # noqa: TRY003 + if binding is not None and binding.binding_name != name: + raise ValueError("replacement binding identity must match") # noqa: TRY003 + done = asyncio.get_running_loop().create_future() + self._binding_changes.put_nowait((name, binding, done)) + self._wake.set() + await asyncio.shield(done) + + async def _apply_binding_changes(self) -> None: + while not self._binding_changes.empty(): + name, binding, done = self._binding_changes.get_nowait() + try: + state = self._families.get(name) + if state is not None: + tasks = [worker.task for worker in state.running.values()] + for task in tasks: + task.cancel() + results = await asyncio.gather(*tasks, return_exceptions=True) + for result in results: + if isinstance(result, _WorkerTerminationError): + raise result # noqa: TRY301 - propagate failed child reaping to leadership revocation + del self._families[name] + if binding is not None: + if any(item.binding.artifact_family == binding.artifact_family for item in self._families.values()): + raise ValueError("an Artifact family already has a binding") # noqa: TRY003, TRY301 + self._families[name] = _FamilyState(binding) + done.set_result(None) + except BaseException as error: + done.set_exception(error) + if isinstance(error, (asyncio.CancelledError, _WorkerTerminationError)): + raise + async def close(self) -> None: self._stop.set() self._wake.set() @@ -666,6 +709,7 @@ async def _run(self) -> None: # noqa: C901 - lifecycle and cancellation boundar while not self._stop.is_set(): self._wake.clear() try: + await self._apply_binding_changes() if self._lease_lost: await self._lose_leadership() if self._fence is None: @@ -694,6 +738,9 @@ async def _run(self) -> None: # noqa: C901 - lifecycle and cancellation boundar except TimeoutError: pass finally: + while not self._binding_changes.empty(): + _, _, done = self._binding_changes.get_nowait() + done.set_exception(RuntimeError("Supervisor stopped before applying the binding")) cleanup = asyncio.create_task( self._lose_leadership(), name=f"artifact-supervisor-cleanup-{self._supervisor_group}" ) @@ -1046,7 +1093,7 @@ def _defer( state.pending_since.setdefault(scope, time.time()) state.retries[scope] = _RetryState(failures, deadline, block, request_generation, probe_error) - async def _dispatch(self, state: _FamilyState) -> None: + async def _dispatch(self, state: _FamilyState) -> None: # noqa: C901 - ordered admission and worker scheduling # Rejections do not occupy slots. Bound their admission work so a page # of slow metadata probes cannot monopolize the shared control loop. loop = asyncio.get_running_loop() @@ -1061,6 +1108,10 @@ async def _dispatch(self, state: _FamilyState) -> None: async with self._database.transaction() as connection: fence = self._require_current_fence() await self._leases.require_fence(connection, fence) + allowed = state.binding.invocation_scope_filter + if allowed is not None and scope not in await allowed(connection, (scope,)): + state.retries.pop(scope, None) + continue row = await self._intents.load(connection, scope, state.binding.binding_name, for_update=True) if row is None or row.requested_generation <= row.handled_generation: state.retries.pop(scope, None) @@ -1649,6 +1700,8 @@ def __init__(self, supervisors: Sequence[ArtifactProcessingSupervisor]) -> None: @property def status(self) -> ArtifactProcessingSupervisorStatus: states = [item.status for item in self.supervisors] + if all(state is ArtifactProcessingSupervisorStatus.DISABLED for state in states): + return ArtifactProcessingSupervisorStatus.DISABLED if ArtifactProcessingSupervisorStatus.DEGRADED in states: return ArtifactProcessingSupervisorStatus.DEGRADED if ArtifactProcessingSupervisorStatus.LEADER in states: @@ -1669,6 +1722,11 @@ def wake(self, binding_name: str | None = None) -> None: for supervisor in self.supervisors: supervisor.wake(binding_name) + async def replace_binding(self, name: str, binding: ArtifactProcessingBinding | None) -> None: + if len(self.supervisors) != 1: + raise ValueError("hot registration requires one global Supervisor") # noqa: TRY003 + await self.supervisors[0].replace_binding(name, binding) + async def close(self) -> None: await asyncio.gather(*(item.close() for item in self.supervisors)) diff --git a/src/powercontext/builtin/runtime/artifact_reads.py b/src/powercontext/builtin/runtime/artifact_reads.py new file mode 100644 index 0000000000..c5d21caee7 --- /dev/null +++ b/src/powercontext/builtin/runtime/artifact_reads.py @@ -0,0 +1,206 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Invocation-scoped Artifact reads and shared custom-family FTS adapters.""" + +from contextlib import AsyncExitStack, asynccontextmanager +from typing import get_args + +from pydantic import JsonValue, RootModel +from sqlalchemy import select + +from powercontext.artifact_plugins import ArtifactReferencePage, ArtifactSnapshot +from powercontext.artifacts import Artifact, ArtifactRef +from powercontext.artifacts.search import ( + ArtifactSearchExecutionContext, +) +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE, ARTIFACT_HEADS_TABLE +from powercontext.builtin.records import BaseArtifactFamily, BaseValueNotFoundError, InvalidBaseAccessRequestError +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService +from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher +from powercontext.server.authz import AccessAction, BuiltinAuthorizationProvider, ResourceRef + + +class _StoredJsonContent(RootModel[dict[str, JsonValue]]): + pass + + +class ArtifactReadSession: + """Constrain reads to one Scope and record exact bodies eligible for generation lineage.""" + + def __init__( + self, + database, + scope_id, + families, + *, + access=None, + principal=None, + audit=None, + guard=None, + embedding_model=None, + search_service=None, + ): + self.database, self.scope_id = database, scope_id + self.families = frozenset(families) + self.access, self.principal, self.audit, self.guard = access, principal, audit, guard + # Only the exact built-in Provider guarantees Scope-to-Artifact read inheritance. + self._scope_covers_artifacts = access is None or type(access.provider) is BuiltinAuthorizationProvider + self.refs: set[tuple[str, str, int]] = set() + self.cursor = SignedCursorCodec() + self.search_service = search_service or ArtifactSearchService( + known_families=get_args(BaseArtifactFamily), + fallback=lambda family: CustomArtifactSearcher(database, family, embedding_model), + ) + + def _family(self, family): + if family not in self.families: + raise InvalidBaseAccessRequestError("family", "is outside the invocation's declared read families") + + @asynccontextmanager + async def _connection(self, family): + self._family(family) + async with AsyncExitStack() as resources: + if self.access is not None: + await resources.enter_async_context(self.access.defer_decision_audit()) + connection = await resources.enter_async_context(self.database.transaction()) + await connection.exec_driver_sql("SAVEPOINT pc_artifact_read") + if self.guard is not None: + await self.guard(connection) + # The Worker guard already checks current Scope contribution in this transaction. + if self.access is not None and (self.guard is None or not self._scope_covers_artifacts): + await self.access.for_read(connection).require( + self.principal, + AccessAction.SCOPE_READ, + ResourceRef.scope(self.scope_id), + context=self.audit, + ) + yield connection + + async def _repository(self, connection, family): + known = await connection.scalar( + select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family) + .where( + ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family == family, + ) + .limit(1) + ) + if known is not None: + return custom_artifact_repository(family) + if family in get_args(BaseArtifactFamily): + artifact_type = type("StoredArtifact", (Artifact[_StoredJsonContent],), {"family": family}) + return ArtifactRepository((artifact_type,)) + raise BaseValueNotFoundError("artifact_family", family) + + async def read(self, family, artifact_id, revision): + async with self._connection(family) as connection: + repository = await self._repository(connection, family) + try: + artifact = ( + await repository.latest(connection, self.scope_id, family, artifact_id) + if revision is None + else await repository.get( + connection, + self.scope_id, + ArtifactRef(family=family, artifact_id=artifact_id, revision=revision), + ) + ) + except RepositoryNotFoundError: + if revision is None: + return None + raise BaseValueNotFoundError("artifact_revision", (family, artifact_id, revision)) from None + if self.access is not None and not self._scope_covers_artifacts and family != "topic-memory": + await self.access.for_read(connection).require( + self.principal, + AccessAction.ARTIFACT_READ, + ResourceRef.artifact(self.scope_id, family=family, artifact_id=artifact_id), + context=self.audit, + ) + ref = artifact.as_ref() + self.refs.add((ref.family, ref.artifact_id, ref.revision)) + custom = isinstance(artifact.content, StoredArtifactContent) + return ArtifactSnapshot( + ref=ref, + lineage=artifact.lineage, + schema_version=artifact.content.schema_version if custom else None, + content=artifact.content.content if custom else artifact.content.model_dump(mode="json", by_alias=True), + source_refs=artifact.lineage.sources, + artifact_refs=artifact.lineage.artifacts, + ) + + async def list(self, family, *, limit=50, cursor=None): + if type(limit) is not int or not 1 <= limit <= 100: + raise InvalidBaseAccessRequestError("limit", "must be between 1 and 100") + expected = {"scope": self.scope_id, "family": family, "limit": limit} + after = self.cursor.after_text(cursor, expected) + selected = [] + async with self._connection(family) as connection: + await self._repository(connection, family) + bound = None if self.access is None else self.access.for_read(connection) + while len(selected) <= limit: + rows = ( + await connection.execute( + select(ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision) + .where( + ARTIFACT_HEADS_TABLE.c.scope_id == self.scope_id, + ARTIFACT_HEADS_TABLE.c.family == family, + ARTIFACT_HEADS_TABLE.c.lifecycle_state == "active", + ARTIFACT_HEADS_TABLE.c.artifact_id > after, + ) + .order_by(ARTIFACT_HEADS_TABLE.c.artifact_id) + .limit(100) + ) + ).all() + if not rows: + break + for artifact_id, revision in rows: + after = artifact_id + if bound is not None and not self._scope_covers_artifacts and family != "topic-memory": + decision = await bound.check( + self.principal, + AccessAction.ARTIFACT_READ, + ResourceRef.artifact(self.scope_id, family=family, artifact_id=artifact_id), + context=self.audit, + ) + if not decision.allowed: + continue + selected.append(ArtifactRef(family=family, artifact_id=artifact_id, revision=revision)) + if len(selected) > limit: + break + items = tuple(selected[:limit]) + return ArtifactReferencePage( + items=items, + next_cursor=self.cursor.encode(expected, items[-1].artifact_id) if len(selected) > limit else None, + ) + + async def search(self, family, query=None, **parameters): + self._family(family) + async with self._connection(family): + pass + payload = dict(parameters) + if query is not None: + payload["query"] = query + outcome = await self.search_service.search( + self.scope_id, + family, + payload, + execution_context=ArtifactSearchExecutionContext( + access=self.access, principal=self.principal, audit=self.audit, trusted_local=self.access is None + ), + ) + return outcome diff --git a/src/powercontext/builtin/runtime/artifact_search.py b/src/powercontext/builtin/runtime/artifact_search.py index 226a420e10..ddc5554b3a 100644 --- a/src/powercontext/builtin/runtime/artifact_search.py +++ b/src/powercontext/builtin/runtime/artifact_search.py @@ -66,8 +66,9 @@ async def invoke( class ArtifactSearchService: """Dispatch repository-known Families without knowing their concrete models.""" - def __init__(self, *, known_families: Iterable[str]) -> None: + def __init__(self, *, known_families: Iterable[str], fallback=None) -> None: self._known_families = frozenset(known_families) + self._fallback = fallback self._bindings: dict[str, _SearcherBinding[Any, ArtifactSearchOutcome]] = {} def register( @@ -105,6 +106,8 @@ async def search( def _binding(self, family: str) -> _SearcherBinding[Any, ArtifactSearchOutcome]: if family not in self._known_families: + if self._fallback is not None: + return _SearcherBinding(self._fallback(family), None) raise ArtifactSearchFamilyNotFound(family) binding = self._bindings.get(family) if binding is None: diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index 4c21e9c5a6..c5b21a0af7 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -107,6 +107,7 @@ ensure_supported_atomic_memory_schema, ) from powercontext.builtin.persistence.bootstrap_receipts import BootstrapReceiptRepository +from powercontext.builtin.persistence.custom_artifacts import ensure_artifact_plugin_schema from powercontext.builtin.persistence.experience_index import ensure_artifact_head_searchable_text from powercontext.builtin.persistence.lexical_projection import refresh_lexical_projections from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex @@ -758,21 +759,8 @@ async def run_profile(scope_id, high): if spec.processing_family in registered_families or (spec.family is None and registered_families) ) topic_memory_processing_available = _topic_memory_processing_available(config, processing_bindings) - artifact_search = ArtifactSearchService(known_families=contexts.repositories.artifacts.families) - artifact_search.register( - ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, contexts.experience_index) - ) - artifact_search.register(contexts.list_skills) - topic_memory_searcher = TopicMemorySearcher( - search=contexts.search_topic_memories, - embedding_model=configured_embedding, - observer=topic_memory_search_observer, - capabilities=contexts.topic_memory_index.capabilities, - ) - artifact_search.register(topic_memory_searcher, embedding_purpose=ModelUsagePurpose.TOPIC_MEMORY_RECALL) - artifact_search.register( - AtomicArtifactSearcher(application=contexts.atomic_memory), - embedding_purpose=ModelUsagePurpose.MEMORY_RECALL, + artifact_search, topic_memory_searcher = compose_artifact_search( + contexts, configured_embedding, observer=topic_memory_search_observer ) runtime = await resources.enter_async_context( BuiltinRuntime( @@ -1114,6 +1102,32 @@ async def _open_artifact_processing_supervisor( yield ArtifactProcessingSupervisors(tuple(supervisors)) +def compose_artifact_search(contexts, embedding_model=None, *, observer=None): + """Use the same Family-owned Searchers in Server, SDK and generation Workers.""" + from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher + + artifact_search = ArtifactSearchService( + known_families=contexts.repositories.artifacts.families, + fallback=lambda family: CustomArtifactSearcher(contexts.database, family, embedding_model), + ) + artifact_search.register( + ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, contexts.experience_index) + ) + artifact_search.register(contexts.list_skills) + topic_memory_searcher = TopicMemorySearcher( + search=contexts.search_topic_memories, + embedding_model=embedding_model, + observer=observer, + capabilities=contexts.topic_memory_index.capabilities, + ) + artifact_search.register(topic_memory_searcher, embedding_purpose=ModelUsagePurpose.TOPIC_MEMORY_RECALL) + artifact_search.register( + AtomicArtifactSearcher(application=contexts.atomic_memory), + embedding_purpose=ModelUsagePurpose.MEMORY_RECALL, + ) + return artifact_search, topic_memory_searcher + + @asynccontextmanager async def open_builtin_contexts( # noqa: C901 -- Backend admission and resource lifetimes stay together. config: BuiltinConfig, @@ -1182,6 +1196,7 @@ async def open_builtin_contexts( # noqa: C901 -- Backend admission and resource validate_only=True, ) else: + await ensure_artifact_plugin_schema(connection) await bootstrap_processing_schema(connection, canonical_processing_manifest(config)) await assert_processing_schema_ready(connection, canonical_processing_manifest(config)) await ensure_skill_distribution_schema(connection) @@ -1272,6 +1287,7 @@ async def open_builtin_contexts( # noqa: C901 -- Backend admission and resource connection, topic_index, worker=_topic_memory_worker, recovery=_archive_recovery, validate_only=True ) else: + await ensure_artifact_plugin_schema(connection) await bootstrap_processing_schema(connection, canonical_processing_manifest(config)) await assert_processing_schema_ready(connection, canonical_processing_manifest(config)) await ensure_skill_distribution_schema(connection) diff --git a/src/powercontext/builtin/runtime/custom_artifact_search.py b/src/powercontext/builtin/runtime/custom_artifact_search.py new file mode 100644 index 0000000000..7a076c22fa --- /dev/null +++ b/src/powercontext/builtin/runtime/custom_artifact_search.py @@ -0,0 +1,463 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declaration-driven text/vector/hybrid search returning complete exact Artifacts.""" + +from __future__ import annotations + +import math +import sqlite3 +import struct +from contextlib import AsyncExitStack +from dataclasses import dataclass +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, RootModel, field_validator +from sqlalchemy import bindparam, select, text +from sqlalchemy.exc import SQLAlchemyError + +from powercontext.artifacts import Artifact, ArtifactRef +from powercontext.artifacts.fusion import ( + FusionCandidate, + FusionChannel, + FusionSelection, + RrfParameters, + fuse_rrf, + parse_fusion_parameters, +) +from powercontext.artifacts.indexes import VectorIndex +from powercontext.artifacts.search import ( + ArtifactSearchExecutionContext, + ArtifactSearchFamilyNotFound, + ArtifactSearchMatch, + ArtifactSearchQuery, + ArtifactSearchUnavailable, + ArtifactSearchUnsupported, + ChannelScore, +) +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.artifacts.search import analyze_fts_query, fts_match_query, lexical_search_score +from powercontext.builtin.inference.protocols import embed_query +from powercontext.builtin.persistence.artifact_indexes import ( + DEFINITIONS, + SCHEMAS, + STATES, + entries, + key, + parse_index, + projection_table, + search_declaration, + vector_bytes, +) +from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository +from powercontext.builtin.persistence.sqlite.profile import ensure_sqlite_vector_extension +from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACT_PROCESSING_BINDING_STATES_TABLE +from powercontext.server.authz import AccessAction, BuiltinAuthorizationProvider, ResourceRef + + +class ArtifactQueryVector(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, allow_inf_nan=False) + profile: EmbeddingProfile + vector: tuple[float, ...] = Field(min_length=1) + + @field_validator("profile", mode="before") + @classmethod + def existing_profile_contract(cls, value): + if isinstance(value, dict) and set(value) - set(EmbeddingProfile.model_fields): + raise ValueError("query vector profile contains unsupported fields") # noqa: TRY003 + return value + + +class CustomArtifactSearchRequest(ArtifactSearchQuery): + query: str | None = Field(default=None, min_length=1, max_length=8192) + mode: Literal["text", "vector", "hybrid"] | None = None + query_vector: ArtifactQueryVector | None = None + indexes: dict[Literal["text", "vector"], tuple[str, ...]] | None = None + min_score: float = Field(default=0, ge=0, le=1, allow_inf_nan=False) + fusion: FusionSelection | None = None + + +@dataclass(frozen=True) +class CustomArtifactSearchOutcome: + matches: tuple[ArtifactSearchMatch, ...] + artifacts: tuple[Any, ...] + + +class CustomArtifactSearcher: + request_type = CustomArtifactSearchRequest + + def __init__(self, database, family, embedding_model=None): + self.database, self.family, self.embedding_model = database, family, embedding_model + self.artifact_type = type( + "CustomSearchArtifact", (Artifact[RootModel[dict[str, JsonValue]]],), {"family": family} + ) + + async def _configuration(self, connection, scope, request, *, config=None): # noqa: C901 - Validate one target or empty-scope snapshot. + durable = ( + ( + await connection.execute( + select(SCHEMAS.c.declaration) + .where(SCHEMAS.c.family == self.family) + .order_by(SCHEMAS.c.schema_version) + ) + ) + .scalars() + .all() + ) + if not durable: + raise ArtifactSearchFamilyNotFound(self.family) + declarations = [declaration for row in durable if (declaration := search_declaration(row)) is not None] + if not declarations: + raise ArtifactSearchUnsupported(self.family, field="family") + state = ( + (await connection.execute(select(STATES).where(key(STATES, scope, self.family)))).mappings().one_or_none() + ) + manifest = {} if state is None else state["manifest"] + if config is None and not manifest.get("config"): + binding = ARTIFACT_PROCESSING_BINDING_STATES_TABLE + definition = await connection.scalar( + select(binding.c.plugin_spec).where(binding.c.plugin_family == self.family) + ) + default = None if definition is None else search_declaration(definition["declaration"]) + config = default or declarations[0] + elif config is None: + # The manifest owns build identity/progress; query policy comes from the + # durable declaration in the same read snapshot, including during repair. + config = next( + (search_declaration(row) for row in durable if row["schema_version"] == manifest["default_schema"]), + None, + ) + if config is None: + raise ArtifactSearchUnavailable(self.family, "index declaration unavailable") + mode = request.mode or config.default_mode + self._validate_input(request, mode) + if mode not in config.modes: + raise ArtifactSearchUnsupported(self.family, field="mode") + try: + chosen = config.select_indexes(mode, request.indexes) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="indexes") from error + profiles = [index.profile for index in chosen if isinstance(index, VectorIndex)] + if profiles and any(profile != profiles[0] for profile in profiles): + raise ArtifactSearchUnsupported(self.family, field="indexes") + profile = profiles[0] if profiles else None + if request.query_vector is not None: + if request.query_vector.profile != profile: + raise ArtifactSearchUnsupported(self.family, field="query_vector.profile") + try: + vector_bytes(request.query_vector.vector, profile) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="query_vector.vector") from error + if request.fusion is not None: + try: + params = parse_fusion_parameters(request.fusion) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="fusion") from error + weights = {"text": config.text_weight, "vector": config.vector_weight} + if set(params.weights) - set(weights): + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + weights.update(params.weights) + if sum(weights.values()) <= 0: + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + if state is None: + has_content = await connection.scalar( + select(ARTIFACT_HEADS_TABLE.c.artifact_id) + .where(ARTIFACT_HEADS_TABLE.c.scope_id == scope, ARTIFACT_HEADS_TABLE.c.family == self.family) + .limit(1) + ) + if mode != "text" or has_content is not None: + raise ArtifactSearchUnavailable(self.family, "index has not been built") + return None, config, mode, (), None + if state["state"] != "ready": + raise ArtifactSearchUnavailable(self.family, "index rebuild is incomplete") + selected = await self._selected_indexes(connection, state, chosen) + if any(isinstance(index, VectorIndex) and index.profile != profile for _, _, index in selected): + raise ArtifactSearchUnsupported(self.family, field="indexes") + return state, config, mode, selected, profile + + async def _selected_indexes(self, connection, state, chosen): + kinds = {index.name: index.kind for index in chosen} + selected = [] + for _, entry in entries(state["manifest"]): + definition = dict( + ( + await connection.execute( + select(DEFINITIONS).where(DEFINITIONS.c.index_def_id == entry["index_def_id"]) + ) + ) + .mappings() + .one() + ) + if definition["index_name"] not in kinds: + continue + await projection_table(connection, definition) + index = parse_index(definition) + if index.kind != kinds[index.name]: + raise ArtifactSearchUnsupported(self.family, field="indexes") + selected.append((state, definition, index)) + if not set(kinds).issubset({definition["index_name"] for _, definition, _ in selected}): + raise ArtifactSearchUnavailable(self.family, "configuration lacks a selected index") + return selected + + async def search(self, scope_id, request, /, *, execution_context=None): + try: + return await self._search(scope_id, request, execution_context=execution_context) + except SQLAlchemyError as error: + raise ArtifactSearchUnavailable(self.family, "search storage is unavailable") from error + + async def _search(self, scope_id, request, /, *, execution_context=None): + execution = execution_context or ArtifactSearchExecutionContext() + access = execution.access + # Pin query policy across encoding outside the transaction; recheck build and access below. + async with AsyncExitStack() as resources: + if access is not None: + await resources.enter_async_context(access.defer_decision_audit()) + async with self.database.transaction(consistent_snapshot=True) as connection: + if access is not None: + await access.for_read(connection).require( + execution.principal, + AccessAction.SCOPE_READ, + ResourceRef.scope(scope_id), + context=execution.audit, + ) + state, config, mode, selected, profile = await self._configuration(connection, scope_id, request) + if state is None: + return CustomArtifactSearchOutcome((), ()) + query_vector = await self._encode_query(request, profile) + async with self.database.transaction(consistent_snapshot=True) as connection: + current, config, mode, selected, checked_profile = await self._configuration( + connection, scope_id, request, config=config + ) + if current["build_id"] != state["build_id"] or checked_profile != profile: + raise ArtifactSearchUnavailable(self.family, "search build changed during query preparation; retry") + bound = None if access is None else access.for_read(connection) + if bound is not None: + await bound.require( + execution.principal, + AccessAction.SCOPE_READ, + ResourceRef.scope(scope_id), + context=execution.audit, + ) + channels = await self._candidates( + connection, scope_id, request, mode, selected, query_vector, profile, bound, execution + ) + native_text = connection.dialect.name != "sqlite" + text_metric = "oceanbase_match" if native_text else "sqlite_bm25" + scored = self._rank(channels, config, mode, request, native_text, text_metric) + return await self._outcome(connection, scope_id, request, channels, scored, native_text, text_metric) + + async def _encode_query(self, request, profile): + query_vector = None + if profile is not None: + if request.query_vector is not None: + try: + query_vector = vector_bytes(request.query_vector.vector, profile) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="query_vector.vector") from error + else: + model = self.embedding_model + if model is None or model.profile != profile: + raise ArtifactSearchUnavailable(self.family, "query encoder unavailable for declared profile") + try: + result = await embed_query(model, (request.query,)) + query_vector = vector_bytes(result.vectors[0], profile) + except Exception as error: + raise ArtifactSearchUnavailable(self.family, "query encoder failed for declared profile") from error + if len(result.vectors) != 1: + raise ArtifactSearchUnavailable(self.family, "query encoder returned an invalid vector count") + return query_vector + + async def _candidates(self, connection, scope_id, request, mode, selected, query_vector, profile, bound, execution): + visible = {} + channels = {name: {} for name in ({"text", "vector"} if mode == "hybrid" else {mode})} + for state, definition, index in selected: + if index.kind == "text": + rows = await self._text( + connection, scope_id, state["build_id"], definition["index_def_id"], request.query + ) + else: + rows = await self._vector(connection, scope_id, definition, state["build_id"], query_vector, profile) + for identity, revision, raw in rows: + if identity not in visible: + visible[identity] = ( + bound is None + # The same transaction already authorized this Scope's reads. + or type(bound.provider) is BuiltinAuthorizationProvider + or ( + await bound.check( + execution.principal, + AccessAction.ARTIFACT_READ, + ResourceRef.artifact(scope_id, family=self.family, artifact_id=identity), + context=execution.audit, + ) + ).allowed + ) + if not visible[identity]: + continue + if not math.isfinite(raw): + raise ArtifactSearchUnavailable(self.family, "index returned a nonfinite score") + address = (identity, revision) + previous = channels[index.kind].get(address) + if previous is None or raw < previous: + channels[index.kind][address] = raw + return channels + + def _rank(self, channels, config, mode, request, native_text, text_metric): + budget = max(request.limit, config.candidate_limit) + ordered = { + name: sorted(values, key=lambda address: (values[address], address))[:budget] + for name, values in channels.items() + } + if mode == "hybrid": + params = RrfParameters(rank_constant=config.rrf_k) + weights = {"text": config.text_weight, "vector": config.vector_weight} + if request.fusion is not None: + try: + params = parse_fusion_parameters(request.fusion) + except ValueError as error: + raise ArtifactSearchUnsupported(self.family, field="fusion") from error + if set(params.weights) - set(weights): + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + weights.update(params.weights) + if sum(weights.values()) <= 0: + raise ArtifactSearchUnsupported(self.family, field="fusion.weights") + fused = fuse_rrf( + tuple( + FusionChannel( + name, + weights[name], + tuple(FusionCandidate(identity, rank) for rank, identity in enumerate(identities, 1)), + ) + for name, identities in ordered.items() + ), + params, + tie_break=lambda identity: identity, + ) + scored = [(hit.key, hit.score) for hit in fused] + elif mode == "text": + scored = [ + (identity, lexical_search_score((-1 if native_text else 1) * channels[mode][identity], text_metric)[0]) + for identity in ordered[mode] + ] + else: + scored = [(identity, 1 / (1 + channels[mode][identity])) for identity in ordered[mode]] + return scored + + async def _outcome(self, connection, scope_id, request, channels, scored, native_text, text_metric): + repository = custom_artifact_repository(self.family) + matches = [] + final = [(identity, score) for identity, score in scored if score >= request.min_score][: request.limit] + artifacts = await repository.get_many( + connection, + scope_id, + tuple( + ArtifactRef(family=self.family, artifact_id=identity, revision=revision) + for (identity, revision), score in final + ), + ) + for ((identity, revision), score), artifact in zip(final, artifacts, strict=True): + raw_scores = { + name: ChannelScore( + (-1 if native_text and name == "text" else 1) * values[(identity, revision)], + text_metric if name == "text" else "l2_distance", + native_text and name == "text", + ) + for name, values in channels.items() + if (identity, revision) in values + } + matches.append( + ArtifactSearchMatch(artifact.as_ref(), score, raw_scores if request.include_scores else None) + ) + public_artifacts = tuple( + self.artifact_type( + artifact_id=artifact.artifact_id, + revision=artifact.revision, + content=artifact.content.content, + lineage=artifact.lineage, + ) + for artifact in artifacts + ) + return CustomArtifactSearchOutcome(tuple(matches), public_artifacts) + + def _validate_input(self, request, mode): + if mode in {"text", "hybrid"} and request.query is None: + raise ArtifactSearchUnsupported(self.family, field="query") + if mode == "text" and request.query_vector is not None: + raise ArtifactSearchUnsupported(self.family, field="query_vector") + if mode == "vector" and ((request.query is None) == (request.query_vector is None)): + raise ArtifactSearchUnsupported(self.family, field="query_vector") + if mode != "hybrid" and request.fusion is not None: + raise ArtifactSearchUnsupported(self.family, field="fusion") + + async def _text(self, connection, scope, build_id, index_def_id, query): + expression = fts_match_query(query) + if expression is None: + return [] + if connection.dialect.name != "sqlite": + result = await connection.execute( + text("""SELECT artifact_id, revision, + MATCH(searchable_text) AGAINST(:query) AS score FROM pc_artifact_search_text + WHERE scope_id=:scope AND family=:family AND build_id=:build_id AND index_def_id=:index_def_id + AND MATCH(searchable_text) AGAINST(:query) > 0 ORDER BY score DESC, artifact_id"""), + { + "query": analyze_fts_query(query), + "scope": scope, + "family": self.family, + "build_id": build_id, + "index_def_id": index_def_id, + }, + ) + return [(row.artifact_id, int(row.revision), -float(row.score)) for row in result] + result = await connection.execute( + text("""SELECT artifact_id, revision, bm25(pc_artifact_search_fts) AS score + FROM pc_artifact_search_fts WHERE pc_artifact_search_fts MATCH :query + AND scope_id=:scope AND family=:family AND build_id=:build_id AND index_def_id=:index_def_id + ORDER BY score, artifact_id"""), + { + "query": expression, + "scope": scope, + "family": self.family, + "build_id": build_id, + "index_def_id": index_def_id, + }, + ) + return [(row.artifact_id, int(row.revision), float(row.score)) for row in result] + + async def _vector(self, connection, scope, definition, build_id, vector, profile): + if connection.dialect.name == "sqlite": + try: + await ensure_sqlite_vector_extension(connection) + except (AttributeError, sqlite3.Error) as error: + raise ArtifactSearchUnavailable(self.family, "SQLite vector extension is unavailable") from error + table = await projection_table(connection, definition) + function = "vec_distance_l2" if connection.dialect.name == "sqlite" else "l2_distance" + # Table identity comes only from a validated server-owned space, never the request. + statement = text(f"""SELECT artifact_id, revision, MIN({function}(embedding, :vector)) AS distance + FROM {table.name} WHERE scope_id=:scope AND family=:family AND build_id=:build_id AND space_id=:space AND index_def_id=:index_def_id + GROUP BY artifact_id, revision ORDER BY distance, artifact_id""") # noqa: S608 - Function and table names come only from validated server-owned metadata. + if connection.dialect.name != "sqlite": + vector = list(struct.unpack(f"<{profile.dimension}f", vector)) + statement = statement.bindparams(bindparam("vector", type_=table.c.embedding.type)) + result = await connection.execute( + statement, + { + "vector": vector, + "scope": scope, + "family": self.family, + "build_id": build_id, + "space": definition["space_id"], + "index_def_id": definition["index_def_id"], + }, + ) + return [(row.artifact_id, int(row.revision), float(row.distance)) for row in result] diff --git a/src/powercontext/builtin/tags.py b/src/powercontext/builtin/tags.py index ff1c9626f7..4be736d541 100644 --- a/src/powercontext/builtin/tags.py +++ b/src/powercontext/builtin/tags.py @@ -27,9 +27,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from powercontext.artifacts import ArtifactRef -from powercontext.builtin.records import ArtifactReadFamily, BaseAccessError, InvalidBaseAccessRequestError +from powercontext.builtin.records import BaseAccessError, BaseArtifactFamily, InvalidBaseAccessRequestError -TaggableArtifactFamily = ArtifactReadFamily +TaggableArtifactFamily = BaseArtifactFamily TagMatch = Literal["all", "any"] TagTargetType = Literal["artifact"] diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index 594ef984fb..522109d3cc 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -57,6 +57,8 @@ ArtifactHandleBinding, ArtifactInventoryStatistics, ArtifactPage, + ArtifactPluginPage, + ArtifactPluginState, ArtifactPublication, ArtifactReadFamily, ArtifactReference, @@ -116,6 +118,7 @@ CaptureWorkingNoteRequest, CatalogChangeProposal, CatalogTagResult, + ChangeArtifactPluginStateRequest, ClearScopeBindingRequest, ClearScopeBindingResponse, CodeQueryRequest, @@ -155,6 +158,7 @@ CreateTopicMemoryArtifactRequest, CreateWorkContractRequest, CurrentWorkHandoff, + CustomArtifactFamilySchema, DownloadRemoteSkillPackageRequest, DreamBudget, DreamCandidateRef, @@ -172,6 +176,8 @@ DreamSourceReference, DreamStatus, DreamUsage, + EnableAllArtifactPluginsResponse, + EnableArtifactPluginResult, EnrollRemoteSkillTargetRequest, EntryChange, EntryChangeOperation, @@ -345,6 +351,7 @@ RecurrenceStreak, RegisterSourceDefinitionRequest, RejectCandidateRequest, + ReloadArtifactPluginRequest, RemainingWorkItem, RememberMemoryRequest, RemoteAgentKind, @@ -524,6 +531,8 @@ "ArtifactHandleBinding", "ArtifactInventoryStatistics", "ArtifactPage", + "ArtifactPluginPage", + "ArtifactPluginState", "ArtifactPublication", "ArtifactReadFamily", "ArtifactReference", @@ -583,6 +592,7 @@ "CaptureWorkingNoteRequest", "CatalogChangeProposal", "CatalogTagResult", + "ChangeArtifactPluginStateRequest", "ClearScopeBindingRequest", "ClearScopeBindingResponse", "CodeQueryRequest", @@ -622,6 +632,7 @@ "CreateTopicMemoryArtifactRequest", "CreateWorkContractRequest", "CurrentWorkHandoff", + "CustomArtifactFamilySchema", "DownloadRemoteSkillPackageRequest", "DreamBudget", "DreamCandidateRef", @@ -639,6 +650,8 @@ "DreamSourceReference", "DreamStatus", "DreamUsage", + "EnableAllArtifactPluginsResponse", + "EnableArtifactPluginResult", "EnrollRemoteSkillTargetRequest", "EntryChange", "EntryChangeOperation", @@ -812,6 +825,7 @@ "RecurrenceStreak", "RegisterSourceDefinitionRequest", "RejectCandidateRequest", + "ReloadArtifactPluginRequest", "RemainingWorkItem", "RememberMemoryRequest", "RemoteAgentKind", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 95b5ab0779..3c2dd84247 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -685,6 +685,109 @@ class ArtifactDreamingOperationCapability(BaseModel): effect: Effect +class CustomArtifactFamilySchema(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + family: StrictStr + schema_version: StrictStr + content_schema: dict[str, Any] + search_fields: Annotated[ + list[StrictStr], Field(description="Legacy persisted text fields; new plugins use search.") + ] + search: Annotated[ + dict[str, Any] | None, Field(description="Persistent declarative indexes and supported search modes.") + ] = None + + +class ReloadArtifactPluginRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + expected_generation: Annotated[StrictInt, Field(ge=0)] + + +class ChangeArtifactPluginStateRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + expected_generation: Annotated[StrictInt, Field(ge=1)] + + +class DesiredState(StrEnum): + ENABLED = "enabled" + DISABLED = "disabled" + UNLOADED = "unloaded" + + +class Phase(StrEnum): + ACTIVATING = "activating" + ENABLED = "enabled" + STOPPING = "stopping" + DISABLED = "disabled" + UNLOADING = "unloading" + UNLOADED = "unloaded" + FAILED = "failed" + + +class ArtifactPluginState(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + binding_name: StrictStr + plugin_id: StrictStr + family: StrictStr + binding_generation: Annotated[StrictInt, Field(ge=1)] + desired_state: DesiredState + phase: Phase + last_error: Annotated[StrictStr | None, Field(...)] + + +class ArtifactPluginPage(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + items: list[ArtifactPluginState] + + +class Outcome(StrEnum): + ACCEPTED = "accepted" + SKIPPED = "skipped" + FAILED = "failed" + + +class ErrorEnum(StrEnum): + CONFLICT = "conflict" + INVALID_PLUGIN = "invalid_plugin" + UNAVAILABLE = "unavailable" + + +class Error(RootModel[ErrorEnum | None]): + root: Annotated[ + ErrorEnum | None, Field(description="Safe per-plugin failure code; null for accepted or skipped entries.") + ] = None + + +class EnableArtifactPluginResult(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + binding_name: StrictStr + outcome: Outcome + status_url: Annotated[StrictStr, Field(description="Registration status URL for the asynchronous result.")] + state: ArtifactPluginState + error: Annotated[ + Error | None, Field(description="Safe per-plugin failure code; null for accepted or skipped entries.") + ] + + +class EnableAllArtifactPluginsResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + items: list[EnableArtifactPluginResult] + + class Configuration(StrEnum): CONFIGURED = "configured" UNCONFIGURED = "unconfigured" @@ -1635,7 +1738,7 @@ class Validation(StrEnum): UNKNOWN = "unknown" -class Outcome(StrEnum): +class Outcome1(StrEnum): SUCCESS = "success" FAILURE = "failure" UNKNOWN = "unknown" @@ -2127,7 +2230,7 @@ class CodeQueryResponse(RootModel[CodeQueryResult | CodeStatus]): root: CodeQueryResult | CodeStatus -class Outcome1(StrEnum): +class Outcome2(StrEnum): INJECTED = "injected" FAILED = "failed" @@ -2138,7 +2241,7 @@ class RecordBootstrapDeliveryRequest(BaseModel): ) scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] receipt_id: Annotated[StrictStr, Field(max_length=64, min_length=1, pattern="^[\\x21-\\x7E]+$")] - outcome: Outcome1 + outcome: Outcome2 class ContextAssemblyFamily(StrEnum): @@ -2263,11 +2366,57 @@ def _reject_invalid_search_value(cls, value: Any) -> Any: return value +class Distance(StrEnum): + L2 = "l2" + + +class Normalization(StrEnum): + NONE = "none" + UNIT = "unit" + + +class EmbeddingProfile(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + profile_id: Annotated[StrictStr, Field(min_length=1, pattern=".*\\S.*")] + model: Annotated[StrictStr, Field(min_length=1, pattern=".*\\S.*")] + dimension: Annotated[StrictInt, Field(ge=1)] + distance: Distance = Distance.L2 + normalization: Normalization = Normalization.UNIT + + +class QueryVector(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + profile: EmbeddingProfile + vector: Annotated[list[StrictFloat], Field(min_length=1)] + + +class TextItem(RootModel[StrictStr]): + root: Annotated[StrictStr, Field(min_length=1)] + + +class VectorItem(RootModel[StrictStr]): + root: Annotated[StrictStr, Field(min_length=1)] + + +class Indexes(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + text: Annotated[list[TextItem] | None, Field(min_length=1)] = None + vector: Annotated[list[VectorItem] | None, Field(min_length=1)] = None + + class SearchArtifactsRequest(BaseModel): model_config = ConfigDict( extra="forbid", ) - query: Annotated[StrictStr, Field(max_length=8192, min_length=1, pattern=".*\\S.*")] + query: Annotated[StrictStr | None, Field(max_length=8192, min_length=1, pattern=".*\\S.*")] = None + query_vector: QueryVector | None = None + indexes: Indexes | None = None limit: Annotated[StrictInt, Field(ge=1, le=200)] = 10 mode: Annotated[StrictStr | None, Field(min_length=1, pattern=".*\\S.*")] = None filters: dict[str, Any] | None = None @@ -2916,15 +3065,8 @@ class BaseArtifactFamily(StrEnum): TOPIC_MEMORY = "topic-memory" -class ArtifactReadFamily(StrEnum): - MEMORY = "memory" - ATOMIC_MEMORY = "atomic-memory" - EXPERIENCE = "experience" - SKILL = "skill" - HANDOFF = "handoff" - PROFILE = "profile" - PROMPT = "prompt" - TOPIC_MEMORY = "topic-memory" +class ArtifactReadFamily(RootModel[StrictStr]): + root: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[a-z][a-z0-9-]*$")] class StatsPeriod(StrEnum): @@ -3643,7 +3785,7 @@ class SourceInstancesResponse(BaseModel): management: Management | None = None -class Outcome2(StrEnum): +class Outcome3(StrEnum): SAVED = "saved" UNCHANGED = "unchanged" @@ -3652,7 +3794,7 @@ class SourceCollectionResult(BaseModel): model_config = ConfigDict( extra="forbid", ) - outcome: Outcome2 + outcome: Outcome3 source_refs: list[SourceTypeReference] @@ -4103,8 +4245,11 @@ class FlushProfileResponse(BaseModel): class ArtifactCollectionItem(BaseModel): scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] - family: ArtifactReadFamily + family: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[a-z][a-z0-9-]*$")] artifact_id: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[\\x21-\\x7E]+$")] + schema_version: Annotated[ + StrictStr | None, Field(description="Persisted schema version for a custom Artifact family.") + ] = None revision: Annotated[StrictInt, Field(ge=1)] sources: list[SourceTypeReference] artifacts: list[ArtifactReference] @@ -4147,8 +4292,11 @@ class ArtifactRevisionPage(BaseModel): class ArtifactRevision(BaseModel): scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] - family: ArtifactReadFamily + family: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[a-z][a-z0-9-]*$")] artifact_id: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern="^[\\x21-\\x7E]+$")] + schema_version: Annotated[ + StrictStr | None, Field(description="Persisted schema version for a custom Artifact family.") + ] = None revision: Annotated[StrictInt, Field(ge=1)] content: dict[str, Any] sources: list[SourceTypeReference] @@ -4531,7 +4679,7 @@ class RecordSkillUsageRequest(BaseModel): selected: StrictBool invoked: Invoked validation: Validation - outcome: Outcome + outcome: Outcome1 task_source: SourceReference | None = None environment_fingerprint: Annotated[StrictStr | None, Field(pattern="^sha256:[0-9a-f]{64}$")] = None @@ -5261,6 +5409,7 @@ class Capabilities(BaseModel): model_config = ConfigDict( extra="forbid", ) + custom_artifact_schemas: Annotated[list[CustomArtifactFamilySchema], Field(validate_default=True)] = [] prompts: Annotated[dict[str, PromptCapability], Field(validate_default=True)] = {} artifact_dreaming: Annotated[ StrictBool, Field(description="Whether asynchronous Artifact Dream execution is configured.") diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index 1b2b014542..1fbc96290f 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -22,6 +22,8 @@ ArtifactCreated, ArtifactHandleBinding, ArtifactPage, + ArtifactPluginPage, + ArtifactPluginState, ArtifactPublication, ArtifactRevision, ArtifactRevisionPage, @@ -45,6 +47,7 @@ CaptureContentSourceRequest, CaptureContentSourceResponse, CaptureWorkingNoteRequest, + ChangeArtifactPluginStateRequest, ClearScopeBindingRequest, ClearScopeBindingResponse, CodeQueryRequest, @@ -66,6 +69,7 @@ DownloadRemoteSkillPackageRequest, DreamRun, DreamRunPage, + EnableAllArtifactPluginsResponse, EnrollRemoteSkillTargetRequest, ExperienceArtifact, ExternalSkillResolution, @@ -152,6 +156,7 @@ RecordTaskOutcomeRequest, RegisterSourceDefinitionRequest, RejectCandidateRequest, + ReloadArtifactPluginRequest, RememberMemoryRequest, RemoteSkillPublication, RemoteSkillReceiptResponse, @@ -668,6 +673,181 @@ class AccessRequirement(BaseModel): access=AccessRequirement(action="server.observe", resource="server", scope_id_field=None, resolver="static"), ) +LIST_ARTIFACT_PLUGINS = Operation[None, ArtifactPluginPage]( + method="GET", + path="/v1/admin/artifact-plugins", + operation_id="list_artifact_plugins", + request_type=None, + request_location=None, + path_parameters=(), + response_type=ArtifactPluginPage, + success_status=200, + summary="List Artifact plugin registrations", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 200: {"description": "Durable registration states."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +GET_ARTIFACT_PLUGIN = Operation[None, ArtifactPluginState]( + method="GET", + path="/v1/admin/artifact-plugins/{binding_name}", + operation_id="get_artifact_plugin", + request_type=None, + request_location=None, + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=200, + summary="Get an Artifact plugin registration", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 200: {"description": "Durable registration state without plugin configuration secrets."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +ENABLE_ALL_ARTIFACT_PLUGINS = Operation[None, EnableAllArtifactPluginsResponse]( + method="POST", + path="/v1/admin/artifact-plugins/enable-all", + operation_id="enable_all_artifact_plugins", + request_type=None, + request_location=None, + path_parameters=(), + response_type=EnableAllArtifactPluginsResponse, + success_status=200, + summary="Enable the currently disabled Artifact plugins", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 200: {"description": "Per-plugin acceptance, skip or failure results."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +RELOAD_ARTIFACT_PLUGIN = Operation[ReloadArtifactPluginRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/reload", + operation_id="reload_artifact_plugin", + request_type=ReloadArtifactPluginRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Load or update a Worker-loaded Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Definition saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +ENABLE_ARTIFACT_PLUGIN = Operation[ChangeArtifactPluginStateRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/enable", + operation_id="enable_artifact_plugin", + request_type=ChangeArtifactPluginStateRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Enable an Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Target saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +DISABLE_ARTIFACT_PLUGIN = Operation[ChangeArtifactPluginStateRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/disable", + operation_id="disable_artifact_plugin", + request_type=ChangeArtifactPluginStateRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Disable an Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Target saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + +UNLOAD_ARTIFACT_PLUGIN = Operation[ChangeArtifactPluginStateRequest, ArtifactPluginState]( + method="POST", + path="/v1/admin/artifact-plugins/{binding_name}/unload", + operation_id="unload_artifact_plugin", + request_type=ChangeArtifactPluginStateRequest, + request_location="body", + path_parameters=("binding_name",), + response_type=ArtifactPluginState, + success_status=202, + summary="Unload an Artifact plugin", + tags=("artifact-plugins",), + scope_mode="none", + responses={ + 202: { + "description": "Target saved; poll its registration state until transition completes.", + "headers": {"Location": {"description": "Registration status URL.", "schema": {"type": "string"}}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="server.admin", resource="server", scope_id_field=None, resolver="static"), +) + GET_CAPABILITIES = Operation[None, Capabilities]( method="GET", path="/v1/capabilities", diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index 1950852184..13ab7bf75e 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -499,6 +499,319 @@ "x-powercontext-access": {"action": "server.observe", "resource": {"type": "server"}}, } }, + "/v1/admin/artifact-plugins": { + "get": { + "tags": ["artifact-plugins"], + "summary": "List Artifact plugin registrations", + "operationId": "list_artifact_plugins", + "responses": { + "200": { + "description": "Durable registration states.", + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginPage"}} + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + } + }, + "/v1/admin/artifact-plugins/{binding_name}": { + "get": { + "tags": ["artifact-plugins"], + "summary": "Get an Artifact plugin registration", + "operationId": "get_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "responses": { + "200": { + "description": "Durable registration state without plugin configuration secrets.", + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/enable-all": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Enable the currently disabled Artifact plugins", + "description": "Captures " + "registrations and " + "generations at " + "request start, then " + "accepts each " + "eligible enable " + "independently. " + "Enabled, unloaded, " + "failed and " + "transitioning " + "plugins are skipped. " + "Concurrent changes " + "and validation " + "failures are " + "reported per plugin " + "without rolling back " + "successes. Does not " + "discover or reload " + "plugins; poll each " + "status URL for the " + "asynchronous " + "outcome.", + "operationId": "enable_all_artifact_plugins", + "responses": { + "200": { + "description": "Per-plugin acceptance, skip or failure results.", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/EnableAllArtifactPluginsResponse"} + } + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/reload": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Load or update a Worker-loaded Artifact plugin", + "description": "Reads the " + "binding's " + "direct " + "child " + "entry " + "under " + "POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR " + "and fixes " + "its " + "resolved " + "immutable " + "directory. " + "The " + "manifest " + "binding_name " + "must " + "match the " + "directory " + "entry and " + "request " + "path. " + "Saves " + "validated " + "configuration " + "and " + "declarations " + "without " + "creating " + "a " + "pipeline. " + "Existing " + "enabled " + "or " + "disabled " + "plugins " + "keep " + "their " + "state; " + "initial, " + "unloaded " + "or failed " + "registrations " + "become " + "disabled. " + "This " + "initial " + "path " + "supports " + "the " + "global " + "SQLite " + "Supervisor. " + "Plugin " + "code is " + "loaded " + "only in " + "child " + "processes.", + "operationId": "reload_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ReloadArtifactPluginRequest"}} + }, + }, + "responses": { + "202": { + "description": "Definition saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/enable": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Enable an Artifact plugin", + "operationId": "enable_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ChangeArtifactPluginStateRequest"} + } + }, + }, + "responses": { + "202": { + "description": "Target saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/disable": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Disable an Artifact plugin", + "operationId": "disable_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ChangeArtifactPluginStateRequest"} + } + }, + }, + "responses": { + "202": { + "description": "Target saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/admin/artifact-plugins/{binding_name}/unload": { + "post": { + "tags": ["artifact-plugins"], + "summary": "Unload an Artifact plugin", + "operationId": "unload_artifact_plugin", + "x-powercontext-access": {"action": "server.admin", "resource": {"type": "server"}}, + "parameters": [ + { + "name": "binding_name", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9-]*$"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ChangeArtifactPluginStateRequest"} + } + }, + }, + "responses": { + "202": { + "description": "Target saved; poll its registration state until transition completes.", + "headers": { + "Location": {"schema": {"type": "string"}, "description": "Registration status URL."} + }, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/ArtifactPluginState"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, "/v1/capabilities": { "get": { "tags": ["capabilities"], @@ -6845,6 +7158,11 @@ "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, "family": {"$ref": "#/components/schemas/ArtifactReadFamily"}, "artifact_id": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": "^[\\x21-\\x7E]+$"}, + "schema_version": { + "type": "string", + "description": "Persisted schema version for a custom Artifact family.", + "nullable": True, + }, "revision": {"type": "integer", "minimum": 1.0}, "sources": {"items": {"$ref": "#/components/schemas/SourceTypeReference"}, "type": "array"}, "artifacts": {"items": {"$ref": "#/components/schemas/ArtifactReference"}, "type": "array"}, @@ -6913,6 +7231,11 @@ "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, "family": {"$ref": "#/components/schemas/ArtifactReadFamily"}, "artifact_id": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": "^[\\x21-\\x7E]+$"}, + "schema_version": { + "type": "string", + "description": "Persisted schema version for a custom Artifact family.", + "nullable": True, + }, "revision": {"type": "integer", "minimum": 1.0}, "content": {"additionalProperties": True, "type": "object"}, "sources": {"items": {"$ref": "#/components/schemas/SourceTypeReference"}, "type": "array"}, @@ -7645,8 +7968,106 @@ "type": "object", "required": ["operation", "output_kind", "effect"], }, + "CustomArtifactFamilySchema": { + "properties": { + "family": {"type": "string"}, + "schema_version": {"type": "string"}, + "content_schema": {"additionalProperties": True, "type": "object"}, + "search_fields": { + "items": {"type": "string"}, + "type": "array", + "description": "Legacy persisted text fields; new plugins use search.", + }, + "search": { + "additionalProperties": True, + "type": "object", + "description": "Persistent declarative indexes and supported search modes.", + }, + }, + "additionalProperties": False, + "type": "object", + "required": ["family", "schema_version", "content_schema", "search_fields"], + }, + "ReloadArtifactPluginRequest": { + "properties": {"expected_generation": {"type": "integer", "minimum": 0.0}}, + "additionalProperties": False, + "type": "object", + "required": ["expected_generation"], + }, + "ChangeArtifactPluginStateRequest": { + "properties": {"expected_generation": {"type": "integer", "minimum": 1.0}}, + "additionalProperties": False, + "type": "object", + "required": ["expected_generation"], + }, + "ArtifactPluginState": { + "properties": { + "binding_name": {"type": "string"}, + "plugin_id": {"type": "string"}, + "family": {"type": "string"}, + "binding_generation": {"type": "integer", "minimum": 1.0}, + "desired_state": {"type": "string", "enum": ["enabled", "disabled", "unloaded"]}, + "phase": { + "type": "string", + "enum": ["activating", "enabled", "stopping", "disabled", "unloading", "unloaded", "failed"], + }, + "last_error": {"type": "string", "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": [ + "binding_name", + "plugin_id", + "family", + "binding_generation", + "desired_state", + "phase", + "last_error", + ], + }, + "ArtifactPluginPage": { + "properties": { + "items": {"items": {"$ref": "#/components/schemas/ArtifactPluginState"}, "type": "array"} + }, + "additionalProperties": False, + "type": "object", + "required": ["items"], + }, + "EnableArtifactPluginResult": { + "properties": { + "binding_name": {"type": "string"}, + "outcome": {"type": "string", "enum": ["accepted", "skipped", "failed"]}, + "status_url": { + "type": "string", + "description": "Registration status URL for the asynchronous result.", + }, + "state": {"$ref": "#/components/schemas/ArtifactPluginState"}, + "error": { + "type": "string", + "enum": ["conflict", "invalid_plugin", "unavailable", None], + "description": "Safe per-plugin failure code; null for accepted or skipped entries.", + "nullable": True, + }, + }, + "additionalProperties": False, + "type": "object", + "required": ["binding_name", "outcome", "status_url", "state", "error"], + }, + "EnableAllArtifactPluginsResponse": { + "properties": { + "items": {"items": {"$ref": "#/components/schemas/EnableArtifactPluginResult"}, "type": "array"} + }, + "additionalProperties": False, + "type": "object", + "required": ["items"], + }, "Capabilities": { "properties": { + "custom_artifact_schemas": { + "items": {"$ref": "#/components/schemas/CustomArtifactFamilySchema"}, + "type": "array", + "default": [], + }, "prompts": { "additionalProperties": {"$ref": "#/components/schemas/PromptCapability"}, "type": "object", @@ -11722,9 +12143,51 @@ "required": ["method"], "x-powercontext-artifact-search-validation": "fusion", }, + "EmbeddingProfile": { + "properties": { + "profile_id": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, + "model": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, + "dimension": {"type": "integer", "minimum": 1.0}, + "distance": {"type": "string", "enum": ["l2"], "default": "l2"}, + "normalization": {"type": "string", "enum": ["none", "unit"], "default": "unit"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["profile_id", "model", "dimension"], + "description": "Existing embedding space identity " + "shared with Memory; no additional " + "modality or version fields.", + }, "SearchArtifactsRequest": { "properties": { "query": {"type": "string", "maxLength": 8192, "minLength": 1, "pattern": ".*\\S.*"}, + "query_vector": { + "properties": { + "profile": {"$ref": "#/components/schemas/EmbeddingProfile"}, + "vector": {"items": {"type": "number"}, "type": "array", "minItems": 1}, + }, + "additionalProperties": False, + "type": "object", + "required": ["profile", "vector"], + }, + "indexes": { + "properties": { + "text": { + "items": {"type": "string", "minLength": 1}, + "type": "array", + "minItems": 1, + "uniqueItems": True, + }, + "vector": { + "items": {"type": "string", "minLength": 1}, + "type": "array", + "minItems": 1, + "uniqueItems": True, + }, + }, + "additionalProperties": False, + "type": "object", + }, "limit": {"type": "integer", "maximum": 200.0, "minimum": 1.0, "default": 10}, "mode": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, "filters": {"additionalProperties": True, "type": "object"}, @@ -11736,7 +12199,6 @@ }, "additionalProperties": False, "type": "object", - "required": ["query"], "x-powercontext-artifact-search-validation": "request", }, "ArtifactChannelScore": { @@ -12736,19 +13198,7 @@ "topic-memory", ], }, - "ArtifactReadFamily": { - "type": "string", - "enum": [ - "memory", - "atomic-memory", - "experience", - "skill", - "handoff", - "profile", - "prompt", - "topic-memory", - ], - }, + "ArtifactReadFamily": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": "^[a-z][a-z0-9-]*$"}, "StatsPeriod": {"type": "string", "enum": ["today", "7d", "30d"]}, "CandidateFamily": { "type": "string", diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index 4db64faf85..071ee59903 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -52,6 +52,7 @@ from powercontext.artifacts.search import ( ArtifactSearchExecutionContext, ArtifactSearchFamilyNotFound, + ArtifactSearchUnavailable, ArtifactSearchUnsupported, ) from powercontext.builtin.artifacts.atomic_memory.errors import ( @@ -416,6 +417,8 @@ ArtifactFamilyAccessCapability, ArtifactHandleBinding, ArtifactPage, + ArtifactPluginPage, + ArtifactPluginState, ArtifactReadFamily, ArtifactReference, ArtifactRevision, @@ -433,6 +436,7 @@ CaptureContentSourceRequest, CaptureContentSourceResponse, CaptureWorkingNoteRequest, + ChangeArtifactPluginStateRequest, ClearScopeBindingRequest, ClearScopeBindingResponse, CodeQueryRequest, @@ -455,6 +459,7 @@ DownloadRemoteSkillPackageRequest, DreamRun, DreamRunPage, + EnableAllArtifactPluginsResponse, EnrollRemoteSkillTargetRequest, ErrorDetail, ErrorResponse, @@ -536,6 +541,7 @@ RecordTaskOutcomeRequest, RegisterSourceDefinitionRequest, RejectCandidateRequest, + ReloadArtifactPluginRequest, RememberMemoryRequest, RemoteSkillAction, RemoteSkillPublication, @@ -696,8 +702,11 @@ CREATE_SOURCE, CREATE_SUBJECT_SOURCE, CREATE_WORK_CONTRACT, + DISABLE_ARTIFACT_PLUGIN, DOWNLOAD_REMOTE_SKILL_PACKAGE, DOWNLOAD_SKILL_PACKAGE, + ENABLE_ALL_ARTIFACT_PLUGINS, + ENABLE_ARTIFACT_PLUGIN, ENROLL_REMOTE_SKILL_TARGET, FINALIZE_HANDOFF, FLUSH_MEMORY, @@ -708,6 +717,7 @@ GENERATE_SKILL, GET_ACCESS_PRINCIPAL, GET_ARTIFACT, + GET_ARTIFACT_PLUGIN, GET_ARTIFACT_REVISION, GET_ARTIFACT_TAGS, GET_CANDIDATE, @@ -740,6 +750,7 @@ LIST_ACCESS_BINDINGS, LIST_ACCESS_RESOURCES, LIST_ACCESS_ROLES, + LIST_ARTIFACT_PLUGINS, LIST_ARTIFACT_REVISIONS, LIST_ARTIFACTS, LIST_CANDIDATES, @@ -771,6 +782,7 @@ RECORD_TASK_OUTCOME, REGISTER_SOURCE_DEFINITION, REJECT_CANDIDATE, + RELOAD_ARTIFACT_PLUGIN, REMEMBER_MEMORY, RENAME_REMOTE_SKILL_TARGET, REPLACE_ACCESS_BINDING, @@ -794,6 +806,7 @@ SET_DEFAULT_SCOPE, SET_SCOPE_BINDING, SUBMIT_SOURCE_OBSERVATION, + UNLOAD_ARTIFACT_PLUGIN, UNPUBLISH_REMOTE_SKILL, UPDATE_SCOPE, UPDATE_SKILL_LIFECYCLE, @@ -802,6 +815,7 @@ ) from powercontext.http._generated.schema import OPENAPI_SCHEMA from powercontext.server import mapping +from powercontext.server.artifact_plugins import ArtifactPluginManager from powercontext.server.atomic_memory import add_atomic_memory_routes from powercontext.server.atomic_memory import record_response as atomic_record_response from powercontext.server.authentication import AuthenticationProvider @@ -847,7 +861,11 @@ owner_recovery_session, owner_write_access, ) -from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES, artifact_family_profile +from powercontext.server.authz.profiles import ( + ARTIFACT_FAMILY_PROFILES, + artifact_family_profile, + register_custom_artifact_access_family, +) from powercontext.server.authz.service import _require_allowed from powercontext.server.context import ( bind_request_id, @@ -1365,6 +1383,9 @@ def create_app( middleware=list(middleware), ) app.openapi_version = OPENAPI_VERSION + app.state.artifact_plugin_manager = None + app.state.artifact_declaration_provider = None + app.state.artifact_access_refresher = None app.state.application = application app.state.capability_provider = capability_provider app.state.readiness_probe = readiness_probe @@ -1463,6 +1484,13 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: _add_route(app, GET_READINESS, get_readiness) _add_route(app, GET_SERVER_INFO, get_server_info) _add_route(app, GET_CAPABILITIES, get_capabilities) + _add_route(app, LIST_ARTIFACT_PLUGINS, list_artifact_plugins) + _add_route(app, ENABLE_ALL_ARTIFACT_PLUGINS, enable_all_artifact_plugins) + _add_route(app, GET_ARTIFACT_PLUGIN, get_artifact_plugin) + _add_route(app, RELOAD_ARTIFACT_PLUGIN, reload_artifact_plugin) + _add_route(app, ENABLE_ARTIFACT_PLUGIN, enable_artifact_plugin) + _add_route(app, DISABLE_ARTIFACT_PLUGIN, disable_artifact_plugin) + _add_route(app, UNLOAD_ARTIFACT_PLUGIN, unload_artifact_plugin) from powercontext.source_management.gateway import mount_source_routes mount_source_routes(app) @@ -1671,6 +1699,7 @@ async def access_readiness_checks(state: Any) -> dict[str, str]: authentication_status = "ready" if (await authentication.readiness()).ready else "not_ready" except Exception: authentication_status = "not_ready" + # This process learns families at startup, on access, or through capabilities discovery. family_capabilities = ",".join( f"{profile.family}:{'enabled' if profile.enabled else 'disabled'}" for profile in sorted(ARTIFACT_FAMILY_PROFILES.values(), key=lambda item: item.family) @@ -1684,11 +1713,95 @@ async def access_readiness_checks(state: Any) -> dict[str, str]: } +_PluginBindingPath = Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")] + + +def _require_artifact_plugin_manager(request: Request) -> ArtifactPluginManager: + manager = request.app.state.artifact_plugin_manager + if manager is None: + raise _RuntimeNotReadyError + return manager + + +async def list_artifact_plugins(http_request: Request) -> ArtifactPluginPage: + return ArtifactPluginPage(items=await _require_artifact_plugin_manager(http_request).list_registrations()) + + +async def enable_all_artifact_plugins(http_request: Request) -> EnableAllArtifactPluginsResponse: + return await _require_artifact_plugin_manager(http_request).enable_all() + + +async def get_artifact_plugin(binding_name: _PluginBindingPath, http_request: Request) -> ArtifactPluginState: + return await _require_artifact_plugin_manager(http_request).get(binding_name) + + +async def reload_artifact_plugin( + binding_name: _PluginBindingPath, + request: ReloadArtifactPluginRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).reload(binding_name, request) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + +async def enable_artifact_plugin( + binding_name: _PluginBindingPath, + request: ChangeArtifactPluginStateRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).change( + binding_name, request.expected_generation, "enabled" + ) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + +async def disable_artifact_plugin( + binding_name: _PluginBindingPath, + request: ChangeArtifactPluginStateRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).change( + binding_name, request.expected_generation, "disabled" + ) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + +async def unload_artifact_plugin( + binding_name: _PluginBindingPath, + request: ChangeArtifactPluginStateRequest, + http_request: Request, + response: Response, +) -> ArtifactPluginState: + result = await _require_artifact_plugin_manager(http_request).change( + binding_name, request.expected_generation, "unloaded" + ) + response.headers["Location"] = f"/v1/admin/artifact-plugins/{binding_name}" + return result + + async def get_capabilities(request: Request) -> Capabilities: capability_provider: CapabilityProvider | None = request.app.state.capability_provider - if capability_provider is not None: - return capability_provider() - return request.app.state.capabilities + value = capability_provider() if capability_provider is not None else request.app.state.capabilities + declaration_provider: Callable[[], Awaitable[list[dict[str, Any]]]] | None = ( + request.app.state.artifact_declaration_provider + ) + if declaration_provider is None: + return value + declarations = await declaration_provider() + for declaration in declarations: + register_custom_artifact_access_family(declaration["family"]) + return Capabilities.model_validate({ + **value.model_dump(mode="json"), + "artifact_families": value.artifact_families + + sorted({item["family"] for item in declarations} - set(value.artifact_families)), + "custom_artifact_schemas": declarations, + }) async def get_server_info(request: Request) -> ServerInfo: @@ -1735,6 +1848,7 @@ async def check_access(payload: AccessCheckRequest, request: Request) -> AccessC ) for requirement in payload.requirements ]) + await _refresh_artifact_access(request.app.state, {resource.family for _, resource in requirements}) decisions = await access.check_batch( _require_principal(), requirements, @@ -1754,6 +1868,8 @@ async def check_access(payload: AccessCheckRequest, request: Request) -> AccessC async def list_access_resources(payload: ListAccessResourcesRequest, request: Request) -> AccessResourcePage: access = _require_access_control(request) resource_type = AccessResourceType(payload.resource_type.value) + if resource_type is AccessResourceType.ARTIFACT: + await _refresh_artifact_access(request.app.state, {payload.family}) page = await access.list_resources( _require_principal(), action=AccessAction(payload.action.value), @@ -1872,6 +1988,7 @@ async def list_access_roles(payload: ListAccessRolesRequest, request: Request) - if payload.family is not None: if resource_type not in {None, AccessResourceType.ARTIFACT}: raise AccessInvalidRequestError("action-resource") + await _refresh_artifact_access(request.app.state, {payload.family}) selected_profile = ARTIFACT_FAMILY_PROFILES.get(payload.family) if selected_profile is None: raise AccessInvalidRequestError("artifact-family") @@ -1925,10 +2042,12 @@ async def list_access_roles(payload: ListAccessRolesRequest, request: Request) - async def list_access_bindings(payload: ListAccessBindingsRequest, request: Request) -> AccessBindingPage: access = _require_access_control(request) + resource = _supported_access_resource(_access_resource(payload.management_resource)) + await _refresh_artifact_access(request.app.state, {resource.family}) page = await access.list_bindings( _require_principal(), BindingSearchRequest( - management_resource=_supported_access_resource(_access_resource(payload.management_resource)), + management_resource=resource, subject=None if payload.subject is None else _access_subject(payload.subject), role=None if payload.role is None else AccessRole(payload.role.value), state=None if payload.state is None else AccessBindingState(payload.state.value), @@ -1945,11 +2064,13 @@ async def list_access_bindings(payload: ListAccessBindingsRequest, request: Requ async def create_access_binding(payload: CreateAccessBindingRequest, request: Request) -> TransportAccessBinding: access = _require_access_control(request) + resource = _supported_access_resource(_access_resource(payload.resource)) + await _refresh_artifact_access(request.app.state, {resource.family}) binding = await access.create_binding( _require_principal(), CreateBinding( subject=_access_subject(payload.subject), - resource=_supported_access_resource(_access_resource(payload.resource)), + resource=resource, role=AccessRole(payload.role.value), idempotency_key=payload.idempotency_key, reason=payload.reason, @@ -1969,6 +2090,7 @@ async def revoke_access_binding(payload: RevokeAccessBindingRequest, request: Re expected_version=payload.expected_version, idempotency_key=payload.idempotency_key, context=_access_audit_context(REVOKE_ACCESS_BINDING.operation_id), + prepare_resource=lambda resource: _refresh_artifact_access(request.app.state, {resource.family}), ) return _access_binding_response(binding) @@ -1989,6 +2111,7 @@ async def replace_access_binding( expires_at=payload.replacement.expires_at, ), context=_access_audit_context(REPLACE_ACCESS_BINDING.operation_id), + prepare_resource=lambda resource: _refresh_artifact_access(request.app.state, {resource.family}), ) return TransportAccessBindingReplacement( previous=_access_binding_response(result.previous), @@ -2638,12 +2761,12 @@ def _list_artifacts_query( async def list_artifacts( scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], request: Annotated[ListArtifactsRequest, Depends(_list_artifacts_query)], application: Annotated[ServerApplication, Depends(_require_application)], ) -> ArtifactPage: result = await application.records.for_scope(scope_id).query_artifacts( - family.value, + family, limit=request.limit, cursor=request.cursor, **( @@ -2725,6 +2848,8 @@ async def search_artifacts( if artifacts is None: raise ArtifactSearchUnsupported(family, field="family") access = access_control_for_mode(http_request.app.state.access_control, mode=http_request.app.state.access_mode) + if access is not None: + await _refresh_artifact_access(http_request.app.state, {family}) execution_context = ArtifactSearchExecutionContext( principal=_require_principal() if access is not None else None, access=access, @@ -2759,13 +2884,13 @@ def _list_artifact_revisions_query( async def list_artifact_revisions( scope_id: _ScopePathId, - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], artifact_id: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$")], request: Annotated[ListArtifactRevisionsRequest, Depends(_list_artifact_revisions_query)], application: Annotated[ServerApplication, Depends(_require_application)], ) -> ArtifactRevisionPage: result = await application.records.for_scope(scope_id).list_artifact_revisions( - family.value, artifact_id, limit=request.limit, cursor=request.cursor + family, artifact_id, limit=request.limit, cursor=request.cursor ) return ArtifactRevisionPage( items=[_artifact_collection_item_response(item) for item in result.items], @@ -2920,7 +3045,7 @@ async def query_artifact_tags( async def get_artifact( scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], artifact_id: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$")], http_request: Request, response: Response, @@ -2931,11 +3056,11 @@ async def get_artifact( ] = None, ) -> ArtifactRevision | Response: result = await application.records.for_scope(scope_id).get_artifact( - family.value, + family, artifact_id, **( {"execution_context": _atomic_execution_context(http_request, GET_ARTIFACT.operation_id)} - if family.value == "atomic-memory" + if family == "atomic-memory" else {} ), ) @@ -2985,19 +3110,19 @@ async def replace_artifact( async def get_artifact_revision( scope_id: Annotated[str, Path(min_length=1, max_length=256, pattern=r".*\S.*")], - family: Annotated[ArtifactReadFamily, Path()], + family: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[a-z][a-z0-9-]*$")], artifact_id: Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[\x21-\x7E]+$")], revision: Annotated[int, Path(ge=1)], http_request: Request, application: Annotated[ServerApplication, Depends(_require_application)], ) -> ArtifactRevision: result = await application.records.for_scope(scope_id).get_artifact_revision( - family.value, + family, artifact_id, revision, **( {"execution_context": _atomic_execution_context(http_request, GET_ARTIFACT_REVISION.operation_id)} - if family.value == "atomic-memory" + if family == "atomic-memory" else {} ), ) @@ -3024,7 +3149,8 @@ def _transport_actor(actor: ArtifactRevisionActor | None) -> TransportAccessPrin def _artifact_revision_response(value: RuntimeArtifactRecord) -> ArtifactRevision: return ArtifactRevision( scope_id=value.scope_id, - family=ArtifactReadFamily(value.family), + family=value.family, + schema_version=value.schema_version, artifact_id=value.artifact_id, revision=value.revision, content=value.content, @@ -3052,7 +3178,8 @@ def _artifact_created_response(value: RuntimeArtifactCreated) -> ArtifactCreated def _artifact_collection_item_response(value: RuntimeArtifactCollectionItem) -> ArtifactCollectionItem: return ArtifactCollectionItem( scope_id=value.scope_id, - family=ArtifactReadFamily(value.family), + family=value.family, + schema_version=value.schema_version, artifact_id=value.artifact_id, revision=value.revision, sources=[mapping.source_type_reference(ref) for ref in value.sources], @@ -4909,6 +5036,15 @@ async def authorize(request: Request) -> None: return authorize +async def _refresh_artifact_access(state: Any, families: set[str | None]) -> None: + refresh = getattr(state, "artifact_access_refresher", None) + if refresh is None: + return + missing = {family for family in families if family is not None and family not in ARTIFACT_FAMILY_PROFILES} + if missing: + await refresh(missing) + + async def _authorize_checks( request: Request, access: AccessControlService, @@ -4916,6 +5052,7 @@ async def _authorize_checks( *, operation: str, ) -> None: + await _refresh_artifact_access(request.app.state, {resource.family for _, resource in checks}) context = _access_audit_context(operation) principal = current_principal() decisions = ( @@ -5217,7 +5354,7 @@ def _path_artifact_family(payload: Mapping[str, Any]) -> str: def _path_artifact_read_family(payload: Mapping[str, Any]) -> str: try: - return ArtifactReadFamily(_nested_request_value(payload, "family")).value + return ArtifactReadFamily(_nested_request_value(payload, "family")).root except ValueError as error: raise AccessInvalidRequestError("artifact-family") from error @@ -5228,7 +5365,16 @@ def _path_artifact_read_access( ) -> tuple[tuple[AccessAction, ResourceRef], ...]: if _path_artifact_read_family(payload) in {BaseArtifactFamily.MEMORY.value, "topic-memory"}: return _path_scope_access(payload, action=AccessAction.SCOPE_READ) - return _path_artifact_access(payload, action=AccessAction.ARTIFACT_READ) + return ( + ( + AccessAction.ARTIFACT_READ, + ResourceRef.artifact( + _nested_request_value(payload, "scope_id"), + family=_path_artifact_read_family(payload), + artifact_id=_nested_request_value(payload, "artifact_id"), + ), + ), + ) def _path_artifact_write_access( @@ -5680,6 +5826,13 @@ def _map_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None]: return 426, error.code, "Upgrade to a client supporting Dream contract 2.", {"required_dream_contract": 2} if isinstance(error, ArtifactSearchFamilyNotFound): return status.HTTP_404_NOT_FOUND, "artifact_family_not_found", "The Artifact Family was not found.", None + if isinstance(error, ArtifactSearchUnavailable): + return ( + status.HTTP_503_SERVICE_UNAVAILABLE, + "artifact_search_unavailable", + "The declared search index or resource is not ready.", + None, + ) if isinstance(error, ArtifactSearchUnsupported): return ( status.HTTP_422_UNPROCESSABLE_CONTENT, diff --git a/src/powercontext/server/artifact_plugins.py b/src/powercontext/server/artifact_plugins.py new file mode 100644 index 0000000000..04fb4f4b19 --- /dev/null +++ b/src/powercontext/server/artifact_plugins.py @@ -0,0 +1,661 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Server-owned hot registration and retained reads for custom Artifact plugins.""" + +from __future__ import annotations + +import asyncio +import logging +import re +from collections.abc import Callable +from contextlib import AsyncExitStack +from datetime import timedelta +from functools import partial +from pathlib import Path +from typing import Any + +from sqlalchemy import insert, select, union, update +from sqlalchemy.exc import IntegrityError, SQLAlchemyError + +from powercontext._logging import log_safely +from powercontext.builtin.artifact_plugin_validation import validate_plugin +from powercontext.builtin.dream.bindings import DREAM_BINDINGS +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.sources import SOURCE_PROCESSING_BINDINGS +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, + ARTIFACT_SEARCH_STATES_TABLE, + ARTIFACT_VECTOR_INDEXES_TABLE, + ARTIFACT_VECTOR_SPACES_TABLE, + SCOPES_TABLE, +) +from powercontext.builtin.records import BaseValueConflictError, BaseValueNotFoundError, InvalidBaseAccessRequestError +from powercontext.builtin.runtime.artifact_plugins import ( + ArtifactPluginWorkerSpec, + run_artifact_plugin_worker, +) +from powercontext.builtin.runtime.artifact_processing import ( + ArtifactProcessingBinding, + ArtifactProcessingSupervisor, + ArtifactProcessingSupervisors, + SpawnArtifactProcessingWorkerLauncher, +) +from powercontext.builtin.runtime.config import DatabaseConfig +from powercontext.builtin.runtime.processing_discovery import SourceProcessingPendingProvider +from powercontext.http._generated.models import ( + ArtifactPluginState, + EnableAllArtifactPluginsResponse, + EnableArtifactPluginResult, + ReloadArtifactPluginRequest, +) +from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES, register_custom_artifact_access_family + +_STABLE_PHASES = frozenset({"enabled", "disabled", "unloaded", "failed"}) +_STATES = ARTIFACT_PROCESSING_BINDING_STATES_TABLE +_STATE_COLUMNS = ( + _STATES.c.binding_name, + _STATES.c.plugin_id, + _STATES.c.plugin_family, + _STATES.c.binding_generation, + _STATES.c.desired_state, + _STATES.c.phase, + _STATES.c.last_error, +) +_SCHEMAS = ARTIFACT_FAMILY_SCHEMAS_TABLE +logger = logging.getLogger(__name__) + + +async def read_artifact_declarations(database: AsyncDatabase) -> list[dict[str, Any]]: + async with database.transaction() as connection: + return list( + ( + await connection.execute( + select(_SCHEMAS.c.declaration).order_by(_SCHEMAS.c.family, _SCHEMAS.c.schema_version) + ) + ).scalars() + ) + + +async def refresh_artifact_access_families(database: AsyncDatabase, families: set[str]) -> None: + families = families - ARTIFACT_FAMILY_PROFILES.keys() + if not families: + return + statement = select(_SCHEMAS.c.family).distinct().where(_SCHEMAS.c.family.in_(families)) + async with database.transaction() as connection: + registered = (await connection.execute(statement)).scalars().all() + for family in registered: + register_custom_artifact_access_family(family) + + +class ArtifactPluginManager: + """Own registrations and transitions; user code stays outside the Server process.""" + + def __init__( + self, + database: AsyncDatabase, + config: DatabaseConfig, + supervisor: ArtifactProcessingSupervisors | None, + security_builder: Callable[[], dict[str, Any] | None], + on_supervisor: Callable[[ArtifactProcessingSupervisors], None], + plugins_dir: Path | None = None, + embedding_model=None, + inference_config=None, + runtime_config=None, + ) -> None: + self.database = database + self.index_builder = ArtifactIndexBuilder(database, embedding_model) + self.inference_config = inference_config + self.runtime_config = runtime_config + self.config = config + self.supervisor = supervisor + self.security_builder = security_builder + self.on_supervisor = on_supervisor + self.plugins_dir = None if plugins_dir is None else plugins_dir.expanduser().resolve() + self._resources = AsyncExitStack() + self._supervisor_lock = asyncio.Lock() + self._locks: dict[str, asyncio.Lock] = {} + self._tasks: set[asyncio.Task[None]] = set() + self._index_locks: dict[str, asyncio.Lock] = {} + self._index_task: asyncio.Task[None] | None = None + self._index_failures: set[tuple[str, str]] = set() + self._index_refresh_families: set[str] = set() + self._space_failures: set[str] = set() + + async def __aenter__(self): + async with self.database.transaction() as connection: + families = (await connection.execute(select(_SCHEMAS.c.family).distinct())).scalars().all() + rows = (await connection.execute(select(_STATES).where(_STATES.c.plugin_id.is_not(None)))).mappings().all() + restored = [] + for persisted in rows: + if persisted["phase"] in {"failed", "disabled", "unloaded"}: + continue + row = dict(persisted) + row["binding_generation"] += 1 + row["phase"] = {"enabled": "activating", "disabled": "stopping", "unloaded": "unloading"}[ + row["desired_state"] + ] + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == row["binding_name"]) + .values(binding_generation=row["binding_generation"], phase=row["phase"]) + ) + restored.append(row) + self._index_refresh_families.update(families) + for row in restored: + await self._transition(row) + await self._discover() + # Retained indexes recover in the background; encoding existing content + # must not delay Server startup or availability of unrelated families. + self._index_task = asyncio.create_task(self._maintain_search(), name="artifact-index-maintenance") + return self + + async def __aexit__(self, *_): + if self._index_task is not None: + self._index_task.cancel() + await asyncio.gather(self._index_task, return_exceptions=True) + try: + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + finally: + await self._resources.aclose() + + async def _ensure_supervisor(self) -> ArtifactProcessingSupervisors: + async with self._supervisor_lock: + if self.supervisor is None: + controller = await self._resources.enter_async_context( + ArtifactProcessingSupervisor( + database=self.database, + bindings=(), + lease_mode="oceanbase" if isinstance(self.config, OceanBaseConfig) else "single-process", + ) + ) + self.supervisor = ArtifactProcessingSupervisors((controller,)) + self.on_supervisor(self.supervisor) + return self.supervisor + + async def _row(self, name: str, *, include_definition: bool = False) -> dict[str, Any]: + columns = (*_STATE_COLUMNS, _STATES.c.plugin_spec) if include_definition else _STATE_COLUMNS + async with self.database.transaction() as connection: + row = ( + ( + await connection.execute( + select(*columns).where(_STATES.c.binding_name == name, _STATES.c.plugin_id.is_not(None)) + ) + ) + .mappings() + .first() + ) + if row is None: + raise BaseValueNotFoundError("artifact_plugin", name) + return dict(row) + + @staticmethod + def _state(row: dict[str, Any]) -> ArtifactPluginState: + return ArtifactPluginState.model_validate({ + "binding_name": row["binding_name"], + "plugin_id": row["plugin_id"], + "family": row["plugin_family"], + "binding_generation": row["binding_generation"], + "desired_state": row["desired_state"], + "phase": row["phase"], + "last_error": row["last_error"], + }) + + async def get(self, name: str) -> ArtifactPluginState: + return self._state(await self._row(name)) + + async def list_registrations(self) -> list[ArtifactPluginState]: + async with self.database.transaction() as connection: + rows = ( + ( + await connection.execute( + select(*_STATE_COLUMNS).where(_STATES.c.plugin_id.is_not(None)).order_by(_STATES.c.binding_name) + ) + ) + .mappings() + .all() + ) + return [self._state(dict(row)) for row in rows] + + async def _discover(self) -> None: + directory = self.plugins_dir + if directory is None: + return + try: + entries = await asyncio.to_thread(lambda: sorted(directory.iterdir())) + except OSError as error: + log_safely( + logger, logging.WARNING, f"Artifact plugin discovery directory unavailable ({type(error).__name__})" + ) + return + known = {state.binding_name for state in await self.list_registrations()} + for entry in entries: + if entry.name in known or not await asyncio.to_thread(entry.is_dir): + continue + try: + await self.reload(entry.name, ReloadArtifactPluginRequest(expected_generation=0)) + except Exception as error: + log_safely( + logger, + logging.WARNING, + f"Artifact plugin discovery failed: {entry.name!r} ({type(error).__name__})", + ) + + async def reload(self, name: str, request: ReloadArtifactPluginRequest) -> ArtifactPluginState: + if re.fullmatch(r"[a-z][a-z0-9-]{0,127}", name) is None: + raise InvalidBaseAccessRequestError("binding_name", "must be a canonical binding name") + if name in {*SOURCE_PROCESSING_BINDINGS, *DREAM_BINDINGS.values()}: + raise InvalidBaseAccessRequestError("binding_name", "is reserved for a built-in processor") + directory = self.plugins_dir + if directory is None: + raise InvalidBaseAccessRequestError("artifact_plugins_dir", "must be configured on the Server") + try: + path = await asyncio.to_thread(lambda: (directory / name).resolve(strict=True)) + is_directory = await asyncio.to_thread(path.is_dir) + except (OSError, RuntimeError) as error: + raise InvalidBaseAccessRequestError( + "binding_name", "must identify a deployed plugin directory entry" + ) from error + if not is_directory: + raise InvalidBaseAccessRequestError("binding_name", "must identify a deployed plugin directory entry") + try: + descriptor = await validate_plugin(str(path)) + except (ValueError, OSError, TimeoutError) as error: + raise InvalidBaseAccessRequestError("plugin", "declaration or configuration validation failed") from error + if descriptor["manifest"]["binding_name"] != name: + raise InvalidBaseAccessRequestError( + "binding_name", "must match plugin.yml, directory entry and request path" + ) + definition = {"directory": str(path), **descriptor} + return await self._accept(name, request.expected_generation, None, definition) + + async def enable_all(self) -> EnableAllArtifactPluginsResponse: + # Freeze eligibility and generations before any per-plugin validation can yield. + snapshot = await self.list_registrations() + results = [] + for state in snapshot: + outcome, error = "skipped", None + current = state + if state.phase == "disabled": + try: + current = await self.change( + state.binding_name, state.binding_generation, "enabled", require_disabled=True + ) + outcome = "accepted" + except Exception as failure: + outcome = "failed" + error = ( + "conflict" + if isinstance(failure, BaseValueConflictError) + else "invalid_plugin" + if isinstance(failure, InvalidBaseAccessRequestError) + else "unavailable" + ) + current = await self.get(state.binding_name) + results.append( + EnableArtifactPluginResult.model_validate({ + "binding_name": state.binding_name, + "outcome": outcome, + "status_url": f"/v1/admin/artifact-plugins/{state.binding_name}", + "state": current, + "error": error, + }) + ) + return EnableAllArtifactPluginsResponse(items=results) + + async def change( + self, name: str, expected: int, target: str, *, require_disabled: bool = False + ) -> ArtifactPluginState: + row = await self._row(name, include_definition=True) + if ( + row["binding_generation"] != expected + or row["phase"] not in _STABLE_PHASES + or (require_disabled and row["phase"] != "disabled") + ): + raise BaseValueConflictError("artifact_plugin", name) + definition = row["plugin_spec"] + if target == "enabled": + if definition is None or row["phase"] == "failed": + raise InvalidBaseAccessRequestError( + "plugin", "an unloaded or failed plugin requires reload before enabling" + ) + if isinstance(self.config, SQLiteConfig) and self.config.is_in_memory: + raise InvalidBaseAccessRequestError( + "database", "Artifact plugin generation requires persistent storage" + ) + try: + descriptor = await validate_plugin(definition["directory"], definition["manifest"]) + except (ValueError, OSError, TimeoutError) as error: + raise InvalidBaseAccessRequestError("plugin", "saved plugin validation failed") from error + if descriptor != {"manifest": definition["manifest"], "declaration": definition["declaration"]}: + raise InvalidBaseAccessRequestError("plugin", "changed code must be registered with reload") + return await self._accept(name, expected, target, definition, require_disabled=require_disabled) + + async def _validate_identity(self, connection, name, old, definition): # noqa: C901 - Validate durable ownership and immutable declarations together. + declaration = None if definition is None else definition["declaration"] + if declaration is None: + if old is None: + raise BaseValueNotFoundError("artifact_plugin", name) + plugin_id, family = old["plugin_id"], old["plugin_family"] + else: + plugin_id, family = declaration["plugin_id"], declaration["family"] + if old is not None and (old["plugin_id"], old["plugin_family"]) != (plugin_id, family): + raise BaseValueConflictError("artifact_plugin_identity", name) + owner = await connection.scalar( + select(_STATES.c.binding_name) + .where(_STATES.c.plugin_family == family, _STATES.c.binding_name != name) + .limit(1) + ) + if owner is not None: + raise BaseValueConflictError("artifact_family", family) + if declaration is not None: + scope_ids = definition["manifest"]["scope_ids"] + if scope_ids is not None: + scopes = set( + ( + await connection.execute( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id.in_(scope_ids)) + ) + ).scalars() + ) + if scopes != set(scope_ids): + raise InvalidBaseAccessRequestError("scope_ids", "must identify existing Scopes") + durable = {key: declaration[key] for key in ("family", "schema_version", "content_schema", "search_fields")} + if "search" in declaration: + durable["search"] = declaration["search"] + from powercontext.builtin.persistence.artifact_indexes import register_definition, search_declaration + + search = search_declaration(durable) + saved = ( + (await connection.execute(select(_SCHEMAS.c.declaration).where(_SCHEMAS.c.family == family))) + .scalars() + .all() + ) + # Only query policy may change; persisted index identity survives unload and restart. + query_policy = { + "modes", + "default_mode", + "defaults", + "candidate_limit", + "rrf_k", + "text_weight", + "vector_weight", + } + fixed = None if search is None else search.model_dump(exclude=query_policy) + for item in saved: + previous_search = search_declaration(item) + previous_fixed = None if previous_search is None else previous_search.model_dump(exclude=query_policy) + if previous_fixed != fixed: + raise BaseValueConflictError("artifact_search_declaration", family) + if ( + saved + and search is not None + and all(item["schema_version"] != durable["schema_version"] for item in saved) + ): + raise BaseValueConflictError("artifact_search_declaration", family) + identity = (_SCHEMAS.c.family == family) & (_SCHEMAS.c.schema_version == durable["schema_version"]) + previous = await connection.scalar(select(_SCHEMAS.c.declaration).where(identity)) + if previous is None: + await connection.execute( + insert(_SCHEMAS).values( + family=family, schema_version=durable["schema_version"], declaration=durable + ) + ) + elif previous["content_schema"] != durable["content_schema"]: + raise BaseValueConflictError("artifact_schema", (family, durable["schema_version"])) + elif previous != durable: + await connection.execute(update(_SCHEMAS).where(identity).values(declaration=durable)) + if search is not None: + for index in search.indexes: + try: + await register_definition(connection, family, durable["schema_version"], index) + except ValueError as error: + raise BaseValueConflictError( + "artifact_search_definition", (family, index.name, index.projection_version) + ) from error + return plugin_id, family + + async def _accept( # noqa: C901 - Registration CAS and lifecycle acceptance share one transaction. + self, + name: str, + expected: int, + target: str | None, + definition: dict[str, Any] | None, + *, + require_disabled: bool = False, + ) -> ArtifactPluginState: + lock = self._locks.setdefault(name, asyncio.Lock()) + if lock.locked(): + raise BaseValueConflictError("artifact_plugin", name) + async with lock: + try: + async with self.database.transaction() as connection: + # This is also the per-registration serialization point used by Worker commit guards. + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == name) + .values(binding_generation=_STATES.c.binding_generation) + ) + old = ( + (await connection.execute(select(_STATES).where(_STATES.c.binding_name == name))) + .mappings() + .first() + ) + generation = 0 if old is None else int(old["binding_generation"]) + if ( + generation != expected + or (old is not None and (old["plugin_id"] is None or old["phase"] not in _STABLE_PHASES)) + or (require_disabled and (old is None or old["phase"] != "disabled")) + ): + raise BaseValueConflictError("artifact_plugin", name) + if target is None: + target = "enabled" if old is not None and old["phase"] == "enabled" else "disabled" + if target == "enabled": + self.security_builder() + if ( + old is not None + and old["plugin_spec"] == definition + and old["desired_state"] == target + and old["phase"] == target + ): + return self._state(dict(old)) + plugin_id, family = await self._validate_identity(connection, name, old, definition) + values = { + "plugin_id": plugin_id, + "plugin_family": family, + "plugin_spec": definition, + "binding_generation": generation + 1, + "desired_state": target, + "phase": {"enabled": "activating", "disabled": "stopping", "unloaded": "unloading"}[target], + "last_error": None, + } + if old is None and target == "disabled": + values["phase"] = "disabled" + if old is None: + await connection.execute(insert(_STATES).values(binding_name=name, **values)) + else: + await connection.execute(update(_STATES).where(_STATES.c.binding_name == name).values(**values)) + except IntegrityError as error: + raise BaseValueConflictError("artifact_plugin", name) from error + register_custom_artifact_access_family(family) + if old is None: + self._index_refresh_families.add(family) + row = {"binding_name": name, **values} + if row["phase"] == target: + return self._state(row) + task = asyncio.create_task(self._transition(row), name=f"artifact-plugin-transition-{name}") + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return self._state(row) + + async def _transition(self, row: dict[str, Any]) -> None: + name, generation = row["binding_name"], row["binding_generation"] + try: + definition = row["plugin_spec"] + target = row["desired_state"] + supervisor = await self._ensure_supervisor() if target == "enabled" else self.supervisor + if supervisor is not None: + await supervisor.replace_binding(name, None) + if target == "enabled" and supervisor is not None: + descriptor = await validate_plugin(definition["directory"], definition["manifest"]) + if descriptor != {"manifest": definition["manifest"], "declaration": definition["declaration"]}: + raise ValueError("plugin code differs from the accepted definition") # noqa: TRY003, TRY301 + manifest, declaration = descriptor["manifest"], descriptor["declaration"] + spec = ArtifactPluginWorkerSpec( + self.config, + name, + generation, + definition, + self.security_builder(), + self.inference_config, + self.runtime_config, + ) + + async def scopes(connection, candidates): + phase = await connection.scalar( + select(_STATES.c.phase).where( + _STATES.c.binding_name == name, _STATES.c.binding_generation == generation + ) + ) + if phase != "enabled": + return frozenset() + scope_ids = manifest["scope_ids"] + return frozenset(candidates if scope_ids is None else set(candidates).intersection(scope_ids)) + + await supervisor.replace_binding( + name, + ArtifactProcessingBinding( + binding_name=name, + artifact_family=declaration["family"], + launcher=SpawnArtifactProcessingWorkerLauncher(partial(run_artifact_plugin_worker, spec)), + max_workers=manifest["processing"]["max_workers"], + worker_timeout_seconds=manifest["processing"]["timeout_seconds"], + automatic_processing_interval=timedelta(seconds=manifest["processing"]["interval_seconds"]), + pending_provider=SourceProcessingPendingProvider(self.database, name, declaration["family"]), + automatic_scope_filter=scopes, + invocation_scope_filter=scopes, + ), + ) + async with self.database.transaction() as connection: + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == name, _STATES.c.binding_generation == generation) + .values(phase=target, plugin_spec=None if target == "unloaded" else definition, last_error=None) + ) + if supervisor is not None: + supervisor.wake(name) + except Exception as error: + # No plugin configuration, credentials or exception input values in management status. + async with self.database.transaction() as connection: + await connection.execute( + update(_STATES) + .where(_STATES.c.binding_name == name, _STATES.c.binding_generation == generation) + .values(phase="failed", last_error=type(error).__name__) + ) + + async def _maintain_search(self): + states = ARTIFACT_SEARCH_STATES_TABLE + while True: + try: + # Persisted unfinished declarations also cover registration by + # another Server, including families with no content or Scope. + async with self.database.transaction() as connection: + spaces = ARTIFACT_VECTOR_SPACES_TABLE + definitions = ARTIFACT_VECTOR_INDEXES_TABLE + unfinished = await connection.scalars( + select(definitions.c.family) + .join(spaces, definitions.c.space_id == spaces.c.space_id) + .where(spaces.c.provision_state == "declared") + .distinct() + ) + self._index_refresh_families.update(unfinished) + # Startup and initial registration discover related pairs needing initialization. + for family in tuple(self._index_refresh_families): + self._index_refresh_families.discard(family) + try: + from powercontext.builtin.persistence.artifact_vector_space import prepare_family_spaces + + await prepare_family_spaces(self.database, family) + self._space_failures.discard(family) + complete = await self._refresh_search(family) + except Exception: + self._index_refresh_families.add(family) + if family not in self._space_failures: + self._space_failures.add(family) + log_safely(logger, logging.WARNING, "Artifact search maintenance unavailable; retrying") + continue + if not complete: + self._index_refresh_families.add(family) + async with self.database.transaction(consistent_snapshot=True) as connection: + pairs = ( + await connection.execute( + select(states.c.scope_id, states.c.family).where(states.c.state != "ready") + ) + ).all() + for scope, family in pairs: + async with self._index_locks.setdefault(family, asyncio.Lock()): + complete = False + try: + from powercontext.builtin.persistence.artifact_vector_space import prepare_family_spaces + + await prepare_family_spaces(self.database, family) + await self.index_builder.refresh(scope, family) + complete = await self.index_builder.build(scope, family) + except Exception: + self._index_failure(scope, family) + if complete: + self._index_failures.discard((scope, family)) + except SQLAlchemyError: + log_safely(logger, logging.WARNING, "Artifact search maintenance unavailable; retrying") + await asyncio.sleep(2) + + def _index_failure(self, scope, family): + if (scope, family) not in self._index_failures: + self._index_failures.add((scope, family)) + log_safely( + logger, logging.WARNING, "Artifact search rebuild incomplete; retrying while search is unavailable" + ) + + async def _refresh_search(self, family): + """Initialize missing targets on startup or first registration, without encoding.""" + async with self._index_locks.setdefault(family, asyncio.Lock()): + async with self.database.transaction(consistent_snapshot=True) as connection: + scopes = ( + ( + await connection.execute( + union( + select(ARTIFACT_HEADS_TABLE.c.scope_id).where(ARTIFACT_HEADS_TABLE.c.family == family), + select(ARTIFACT_SEARCH_STATES_TABLE.c.scope_id).where( + ARTIFACT_SEARCH_STATES_TABLE.c.family == family + ), + ) + ) + ) + .scalars() + .all() + ) + complete = True + for scope in scopes: + try: + await self.index_builder.refresh(scope, family) + except Exception: + complete = False + self._index_failure(scope, family) + return complete diff --git a/src/powercontext/server/authz/profiles.py b/src/powercontext/server/authz/profiles.py index c919f14347..c8d5c41e26 100644 --- a/src/powercontext/server/authz/profiles.py +++ b/src/powercontext/server/authz/profiles.py @@ -140,6 +140,23 @@ def subject_compatibility(self) -> dict[AccessRole, frozenset[str]]: } +def register_custom_artifact_access_family(family: str) -> None: + """Use framework-owned read/write semantics for a persisted custom family.""" + + if family not in ARTIFACT_FAMILY_PROFILES: + ARTIFACT_FAMILY_PROFILES[family] = ArtifactFamilyAccessProfile( + family=family, + enabled=True, + share_unit="artifact", + shareable_states=frozenset(), + base_action=AccessAction.ARTIFACT_READ, + additional_actions=frozenset(), + grantable_roles=frozenset(), + selector="forbidden", + mutation_semantics=frozenset({AccessAction.ARTIFACT_WRITE}), + ) + + def artifact_family_profile(resource: ResourceRef) -> ArtifactFamilyAccessProfile: """Validate an exact Artifact resource and return its enabled profile.""" diff --git a/src/powercontext/server/authz/service.py b/src/powercontext/server/authz/service.py index 0c96ccad6c..5372f4a88b 100644 --- a/src/powercontext/server/authz/service.py +++ b/src/powercontext/server/authz/service.py @@ -1404,6 +1404,7 @@ async def revoke_binding( expected_version: int, idempotency_key: str, context: AccessAuditContext, + prepare_resource: Callable[[ResourceRef], Awaitable[None]] | None = None, ) -> AccessBinding: _validate_idempotency_key(idempotency_key) actor = _required_principal(principal) @@ -1414,6 +1415,8 @@ async def revoke_binding( if decision.allowed: raise AccessBindingNotFoundError raise AccessDeniedError + if prepare_resource is not None: + await prepare_resource(binding.resource) await self.require_any( actor, _administrative_checks(binding.resource, deployment_id=self.deployment_id), @@ -1444,6 +1447,7 @@ async def replace_binding( request: ReplaceBinding, *, context: AccessAuditContext, + prepare_resource: Callable[[ResourceRef], Awaitable[None]] | None = None, ) -> BindingReplacement: actor = _required_principal(principal) current = await _access_call(self._relationship_reader().get_binding(request.binding_id)) @@ -1457,6 +1461,8 @@ async def replace_binding( if decision.allowed: raise AccessBindingNotFoundError raise AccessDeniedError + if prepare_resource is not None: + await prepare_resource(current.resource) validate_binding_subject(current.resource, current.role, request.subject.type) if isinstance(request.subject, GroupRef) and not self.provider_capabilities.group_subjects: raise AccessInvalidRequestError("group-subjects-unavailable") diff --git a/src/powercontext/server/factory.py b/src/powercontext/server/factory.py index 91feecae88..04c10b48f7 100644 --- a/src/powercontext/server/factory.py +++ b/src/powercontext/server/factory.py @@ -20,10 +20,12 @@ import logging from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from contextlib import AsyncExitStack, asynccontextmanager, nullcontext, suppress +from functools import partial from pathlib import Path from fastapi import FastAPI, Request, Response from fastapi.routing import APIRoute +from sqlalchemy import select from starlette.middleware import Middleware from powercontext._logging import log_safely @@ -38,6 +40,7 @@ from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.records import RelationalRecordService from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE from powercontext.builtin.records import BaseOperationNotSupportedError from powercontext.builtin.runtime import ( BuiltinRuntime, @@ -62,6 +65,11 @@ from powercontext.paths import default_scheduler_path from powercontext.server.access import HttpAccessLogMiddleware from powercontext.server.app import _bind_evidence_access, access_readiness_checks, create_app +from powercontext.server.artifact_plugins import ( + ArtifactPluginManager, + read_artifact_declarations, + refresh_artifact_access_families, +) from powercontext.server.authentication import ( AuthenticationProvider, MultiBearerAuthenticationProvider, @@ -83,6 +91,7 @@ owner_recovery_session, owner_write_access, ) +from powercontext.server.authz.profiles import register_custom_artifact_access_family from powercontext.server.authz.repository import RelationalAccessRepository from powercontext.server.context import current_principal, current_request_id from powercontext.server.cursor_secret import resolve_cursor_secret @@ -188,7 +197,7 @@ def create_server_app( # noqa: C901 readiness_probe = _ServerReadinessProbe(metrics, tracing=resolved_tracing) @asynccontextmanager - async def lifespan(app: FastAPI) -> AsyncIterator[None]: + async def lifespan(app: FastAPI) -> AsyncIterator[None]: # noqa: C901 - shared Server and plugin resource lifecycle _log_lifecycle("server.starting", "PowerContext Server is starting") if resolved.allow_insecure_http: _log_insecure_remote_http_warning() @@ -301,7 +310,36 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: identity_repository = ServerIdentityRepository(runtime.primary_database) await identity_repository.initialize() server_id = await identity_repository.load_or_create() + records = await _restore_artifact_access_profiles(runtime) + app.state.artifact_access_refresher = partial(refresh_artifact_access_families, records.database) _bind_evidence_access(runtime, active_access_control, resolved.access.mode) + if ( + config.runtime.artifact_processing_supervisor_mode == "global" + and config.runtime.artifact_processing_role != "api" + ): + + def bind_plugin_supervisor(supervisors) -> None: + runtime.artifact_processing_supervisor = supervisors + + app.state.artifact_plugin_manager = await resources.enter_async_context( + ArtifactPluginManager( + records.database, + config.database, + runtime.artifact_processing_supervisor, + lambda: build_worker_security( + resolved, + active_access_control, + legacy_static_principal=static_principal if legacy_static_admin else None, + enabled=True, + injected=configured_access_control is not None, + ), + on_supervisor=bind_plugin_supervisor, + plugins_dir=resolved.artifact_plugins_dir, + embedding_model=runtime._topic_memory_embedding_model, + inference_config=config.inference, + runtime_config=config.runtime, + ) + ) if active_access_control is not None: migrated, unresolved = await runtime._records().migrate_handoff_receipts( active_access_control.committed_receipt_identity, @@ -332,6 +370,7 @@ def current_capabilities() -> Capabilities: } ) + app.state.artifact_declaration_provider = partial(read_artifact_declarations, records.database) app.state.capabilities = capabilities app.state.capability_provider = current_capabilities app.state.server_info = server_info(server_id) @@ -351,6 +390,9 @@ def current_capabilities() -> Capabilities: await asyncio.gather(deployment_task, return_exceptions=True) _log_lifecycle("server.stopping", "PowerContext Server is stopping") readiness_probe.unbind() + app.state.artifact_plugin_manager = None + app.state.artifact_declaration_provider = None + app.state.artifact_access_refresher = None app.state.application = None app.state.source_gateway = None app.state.dashboard_runtime = None @@ -440,6 +482,18 @@ def current_capabilities() -> Capabilities: return app +async def _restore_artifact_access_profiles(runtime: BuiltinRuntime) -> RelationalRecordService: + """Restore retained family access even when this process runs no plugin manager.""" + records = runtime._records() + if not isinstance(records, RelationalRecordService): + raise TypeError("Artifact plugins require relational record storage") # noqa: TRY003 + async with records.database.transaction() as connection: + families = (await connection.execute(select(ARTIFACT_FAMILY_SCHEMAS_TABLE.c.family).distinct())).scalars().all() + for family in families: + register_custom_artifact_access_family(family) + return records + + async def _source_gateway(runtime: BuiltinRuntime, settings: ServerSettings): from powercontext.source_management.gateway import SourceGateway from powercontext.source_management.write import SourceWritePlugins diff --git a/src/powercontext/server/settings.py b/src/powercontext/server/settings.py index e5ef05518c..9527d43503 100644 --- a/src/powercontext/server/settings.py +++ b/src/powercontext/server/settings.py @@ -247,6 +247,7 @@ class ServerSettings(BaseSettings): handoff_generation_verification_secrets: tuple[SecretStr, ...] = Field(default=(), max_length=8, repr=False) code: CodeConfig = Field(default_factory=CodeConfig) runtime: RuntimeConfig = Field(default_factory=RuntimeConfig) + artifact_plugins_dir: Path | None = None source_plugins_dir: Path | None = None database: DatabaseConfig = Field(default_factory=_default_database, discriminator="kind") handoff_report: HandoffReportConfig = Field(default_factory=HandoffReportConfig) diff --git a/tests/builtin/persistence/test_artifact_indexes.py b/tests/builtin/persistence/test_artifact_indexes.py new file mode 100644 index 0000000000..86ff50a771 --- /dev/null +++ b/tests/builtin/persistence/test_artifact_indexes.py @@ -0,0 +1,834 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Actual SQLite search, exact evidence and atomic projection publication.""" + +import asyncio +from contextlib import asynccontextmanager + +import pytest +from sqlalchemy import insert, select, update + +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, FragmentProjection, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchUnavailable, ArtifactSearchUnsupported +from powercontext.builtin.artifacts.memory.models import EmbeddingProfile +from powercontext.builtin.inference.models import EmbeddingResult +from powercontext.builtin.persistence.artifact_governance import ArtifactGovernanceRepository, ArtifactLifecycleState +from powercontext.builtin.persistence.artifact_indexes import ( + SCHEMAS, + STATES, + ArtifactIndexBuilder, + ArtifactProjectionPublisher, +) +from powercontext.builtin.persistence.artifacts import RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService +from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher + +PROFILE = EmbeddingProfile(profile_id="features-v1", model="fixture", dimension=2, normalization="none") + + +def declaration(*, version="1", vector_field="vector", profile=PROFILE): + return ArtifactSearchDeclaration( + version=version, + indexes=( + TextIndex(name="lexical", fields=("text",), projection_version="1"), + VectorIndex( + name="visual", + projection_version=version, + source="content", + profile=profile, + vector_field=vector_field, + profile_field="profile", + fragments=FragmentProjection(array_field="fragments", id_field="id"), + ), + ), + modes=("text", "vector", "hybrid"), + default_mode="text", + ) + + +@asynccontextmanager +async def store(config=None, model=None): + async with SQLiteProfile.open(config or SQLiteConfig(), tables=BUILTIN_TABLES) as storage: + yield ( + storage.database, + ArtifactIndexBuilder(storage.database, model), + ArtifactProjectionPublisher(storage.database, model), + ) + + +async def register(db, family, search): + durable = {"family": family, "schema_version": "1", "search_fields": [], "search": search.model_dump(mode="json")} + async with db.transaction() as connection: + existing = await connection.scalar(select(SCHEMAS.c.family).where(SCHEMAS.c.family == family)) + if existing is None: + await connection.execute(insert(SCHEMAS).values(family=family, schema_version="1", declaration=durable)) + else: + await connection.execute(update(SCHEMAS).where(SCHEMAS.c.family == family).values(declaration=durable)) + + +async def write(db, publisher, scope, family, identity, content): + prepared = await publisher.prepare(scope, family, "1", content) + repository = custom_artifact_repository(family, prepared={identity: prepared}) + async with db.transaction() as connection: + draft = RepositoryArtifactDraft( + family=family, content=StoredArtifactContent(schema_version="1", content=content) + ) + try: + previous = await repository.latest(connection, scope, family, identity) + except RepositoryNotFoundError: + return await repository.create(connection, scope, identity, draft) + return await repository.revise(connection, scope, previous, draft) + + +def body(text, *vectors, profile=PROFILE): + return { + "text": text, + "fragments": [ + {"id": f"f{i}", "vector": vector, "profile": profile.model_dump(mode="json")} + for i, vector in enumerate(vectors) + ], + } + + +def service(db, family, model=None): + result = ArtifactSearchService(known_families=(family,)) + result.register(CustomArtifactSearcher(db, family, model)) + return result + + +def test_missing_vector_extension_keeps_text_search_and_content_writes_available(monkeypatch, tmp_path): + import sqlite_vec + + async def run(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'vectors.db'}") + async with store(config) as (db, builder, publisher): + await register(db, "reports", declaration()) + await write(db, publisher, "scope", "reports", "a", body("数据库", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + + monkeypatch.setattr(sqlite_vec, "loadable_path", lambda: str(tmp_path / "missing-sqlite-vec")) + async with store(config) as (db, _, publisher): + search = service(db, "reports") + with pytest.raises(ArtifactSearchUnavailable): + await search.search( + "scope", + "reports", + { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + }, + ) + result = await search.search("scope", "reports", {"mode": "text", "query": "数据库"}) + assert [artifact.artifact_id for artifact in result.artifacts] == ["a"] + revised = await write(db, publisher, "scope", "reports", "a", body("数据库 restored", [0, 1])) + assert revised.revision == 2 + + asyncio.run(run()) + + +def test_exact_vectors_collapse_fragments_before_limit_and_preserve_scope_history(): + async def run(): + async with store() as (db, builder, publisher): + for family in ("reports", "other"): + await register(db, family, declaration()) + await write(db, publisher, "scope", "reports", "a", body("数据库 timeout", *([[0, 0]] * 8))) + await write(db, publisher, "scope", "reports", "b", body("数据库 normal", [1, 0])) + await write(db, publisher, "foreign", "reports", "a", body("数据库 hidden", [0, 0])) + await write(db, publisher, "scope", "other", "a", body("数据库 hidden", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + request = { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]}, + "limit": 2, + "include_scores": True, + } + result = await search.search("scope", "reports", request) + assert [artifact.artifact_id for artifact in result.artifacts] == ["a", "b"] + assert result.matches[0].channel_scores["vector"].raw == 0 + old = result.artifacts[0].as_ref() + hybrid = await search.search("scope", "reports", {**request, "mode": "hybrid", "query": "数据库"}) + assert {artifact.artifact_id for artifact in hybrid.artifacts} == {"a", "b"} + assert all(0 <= hit.retrieval_score <= 1 for hit in hybrid.matches) + await write(db, publisher, "scope", "reports", "a", body("recovered")) + current = await search.search("scope", "reports", request) + assert [artifact.artifact_id for artifact in current.artifacts] == ["b"] + async with db.transaction() as connection: + historical = await custom_artifact_repository("reports").get(connection, "scope", old) + assert len(historical.content.content["fragments"]) == 8 + assert not (await search.search("scope", "reports", {"query": "timeout"})).matches + + asyncio.run(run()) + + +def test_initial_build_catches_insert_before_scan_cursor_and_publishes_atomically(): + async def run(): + async with store() as (db, builder, publisher): + + class ConcurrentEncoder: + profile = PROFILE + inserted = False + + async def embed(self, texts): + if not self.inserted: + self.inserted = True + # The text generation has finished its scan. This new identity is + # before its cursor and must be picked up by invalidation replay. + await write(db, publisher, "scope", "reports", "a", {"text": "original", "title": "new-index"}) + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + builder = ArtifactIndexBuilder(db, ConcurrentEncoder()) + initial = ArtifactSearchDeclaration( + version="1", + indexes=( + TextIndex(name="lexical", fields=("title",), projection_version="1"), + VectorIndex( + name="semantic", projection_version="1", source="text", profile=PROFILE, fields=("title",) + ), + ), + modes=("text",), + default_mode="text", + ) + await register(db, "reports", initial) + await write(db, publisher, "scope", "reports", "z", {"text": "original", "title": "new-index"}) + await builder.refresh("scope", "reports") + search = service(db, "reports") + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "original"}) + async with db.transaction() as connection: + await connection.exec_driver_sql( + "CREATE TRIGGER fail_release BEFORE UPDATE OF state ON pc_artifact_search_states WHEN new.state='ready' BEGIN SELECT RAISE(ABORT, 'release failure'); END" + ) + with pytest.raises(Exception, match="release failure"): + await builder.build("scope", "reports") + async with db.transaction() as connection: + assert ( + await connection.scalar( + select(STATES.c.state).where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + ) + == "building" + ) + await connection.exec_driver_sql("DROP TRIGGER fail_release") + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "original"}) + assert await builder.build("scope", "reports") + assert not (await search.search("scope", "reports", {"query": "original"})).matches + assert len((await search.search("scope", "reports", {"query": "new-index"})).matches) == 2 + + asyncio.run(run()) + + +def test_text_vectors_use_declared_encoder_and_unready_profile_never_falls_back(): + class Encoder: + profile = PROFILE + + async def embed(self, texts): + return EmbeddingResult(vectors=tuple((float(len(text)), 1.0) for text in texts)) + + async def run(): + model = Encoder() + async with store(model=model) as (db, builder, publisher): + search_config = ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", projection_version="1", source="text", profile=PROFILE, fields=("text",) + ), + ), + modes=("vector",), + default_mode="vector", + ) + await register(db, "reports", search_config) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库"}) + search = service(db, "reports", model) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "数据库"}) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + assert (await search.search("scope", "reports", {"query": "数据库"})).artifacts[0].artifact_id == "a" + with pytest.raises(ArtifactSearchUnavailable): + await service(db, "reports").search("scope", "reports", {"query": "数据库"}) + assert ( + await service(db, "reports").search( + "scope", "reports", {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [3, 1]}} + ) + ).matches + + asyncio.run(run()) + + +def test_query_errors_are_unsupported_and_declared_indexes_are_unavailable_during_repair(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + await write(db, publisher, "scope", "reports", "a", body("数据库", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + vector = {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]} + invalid = ( + {"query": "数据库", "mode": "text", "indexes": {"vector": ["visual"]}}, + {"query": "数据库", "indexes": {"text": []}}, + { + "query": "数据库", + "mode": "hybrid", + "query_vector": vector, + "fusion": {"method": "rrf", "params": {"weights": {"text": 0, "vector": 0}}}, + }, + {"mode": "vector", "query_vector": {**vector, "vector": [0]}}, + {"mode": "vector", "query": "数据库", "query_vector": vector}, + ) + for request in invalid: + with pytest.raises(ArtifactSearchUnsupported): + await search.search("scope", "reports", request) + await builder.refresh("scope", "reports", rebuild=True) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "数据库", "indexes": {"text": ["lexical"]}}) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", {"query": "数据库"}) + + asyncio.run(run()) + + +def test_governance_removal_restore_and_rebuild_preserve_exact_history(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + original = await write(db, publisher, "scope", "reports", "a", body("数据库", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + async with db.transaction() as connection: + state = await ArtifactGovernanceRepository().transition( + connection, "scope", "reports", "a", 0, ArtifactLifecycleState.DEPRECATED, None + ) + assert not (await search.search("scope", "reports", {"query": "数据库"})).matches + # Restore requires all serving projections to be prepared outside its transaction. + with pytest.raises(ValueError, match="prepared before"): + async with db.transaction() as connection: + await ArtifactGovernanceRepository().transition( + connection, + "scope", + "reports", + "a", + state.governance_generation, + ArtifactLifecycleState.ACTIVE, + None, + ) + prepared = await publisher.prepare("scope", "reports", "1", original.content.content) + async with db.transaction() as connection: + governance = ArtifactGovernanceRepository(prepared_indexes={("scope", "reports", "a"): prepared}) + state = await governance.transition( + connection, + "scope", + "reports", + "a", + state.governance_generation, + ArtifactLifecycleState.ACTIVE, + None, + ) + assert (await search.search("scope", "reports", {"query": "数据库"})).artifacts[0].revision == 1 + await builder.refresh("scope", "reports", rebuild=True) + async with db.transaction() as connection: + await ArtifactGovernanceRepository().transition( + connection, + "scope", + "reports", + "a", + state.governance_generation, + ArtifactLifecycleState.RETIRED, + None, + ) + assert await builder.build("scope", "reports") + assert not (await search.search("scope", "reports", {"query": "数据库"})).matches + async with db.transaction() as connection: + assert ( + await custom_artifact_repository("reports").get(connection, "scope", original.as_ref()) + ).content == original.content + + asyncio.run(run()) + + +def test_incompatible_content_vector_write_preserves_search_and_history(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + first = await write(db, publisher, "scope", "reports", "a", body("数据库", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + next_profile = PROFILE.model_copy(update={"profile_id": "features-v2", "model": "second"}) + with pytest.raises(ValueError, match="profile differs"): + await write(db, publisher, "scope", "reports", "a", body("数据库", [0, 1], profile=next_profile)) + search = service(db, "reports") + request = { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + } + assert (await search.search("scope", "reports", request)).artifacts[0].revision == 1 + async with db.transaction() as connection: + assert ( + await custom_artifact_repository("reports").get(connection, "scope", first.as_ref()) + ).content == first.content + + asyncio.run(run()) + + +def test_initial_content_vector_validation_rejects_bad_record_without_requiring_index_resources(): + async def run(): + async with store() as (db, _, publisher): + base = declaration() + search_config = ArtifactSearchDeclaration( + version="1", + indexes=( + *base.indexes, + VectorIndex( + name="semantic", fields=("text",), source="text", profile=PROFILE, projection_version="1" + ), + ), + modes=base.modes, + default_mode="text", + defaults={ + "text": {"text": ("lexical",)}, + "vector": {"vector": ("visual",)}, + "hybrid": {"text": ("lexical",), "vector": ("visual",)}, + }, + ) + await register(db, "reports", search_config) + wrong = PROFILE.model_copy(update={"profile_id": "features-v2", "model": "second"}) + with pytest.raises(ValueError, match="profile differs"): + await write(db, publisher, "scope", "reports", "a", body("invalid", [1, 0], profile=wrong)) + async with db.transaction() as connection: + with pytest.raises(RepositoryNotFoundError): + await custom_artifact_repository("reports").latest(connection, "scope", "reports", "a") + + # The family is still unready: neither the text encoder nor physical vector spaces are needed for this write. + stored = await write(db, publisher, "scope", "reports", "a", body("valid", [1, 0])) + assert stored.revision == 1 + + asyncio.run(run()) + + +def test_repair_content_vector_validation_rejects_bad_revision_across_ready_transition(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + original = await write(db, publisher, "scope", "reports", "a", body("original", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + await builder.refresh("scope", "reports", rebuild=True) + + with pytest.raises(ValueError, match="dimension"): + await write(db, publisher, "scope", "reports", "a", body("invalid dimension", [1])) + async with db.transaction() as connection: + latest = await custom_artifact_repository("reports").latest(connection, "scope", "reports", "a") + assert latest.revision == original.revision + assert latest.content == original.content + + duplicate = body("duplicate fragments", [1, 0], [0, 1]) + duplicate["fragments"][1]["id"] = duplicate["fragments"][0]["id"] + with pytest.raises(ValueError, match="identities must be unique"): + await write(db, publisher, "scope", "reports", "a", duplicate) + + changed = body("changed while repairing", [0, 1]) + prepared = await publisher.prepare("scope", "reports", "1", changed) + assert await builder.build("scope", "reports") + repository = custom_artifact_repository("reports", prepared={"a": prepared}) + draft = RepositoryArtifactDraft( + family="reports", content=StoredArtifactContent(schema_version="1", content=changed) + ) + with pytest.raises(ValueError): + async with db.transaction() as connection: + await repository.revise(connection, "scope", original, draft) + async with db.transaction() as connection: + latest = await repository.latest(connection, "scope", "reports", "a") + assert latest.revision == original.revision + assert latest.content == original.content + + asyncio.run(run()) + + +def test_vector_profile_is_validated_against_requested_index(): + async def run(): + async with store() as (db, builder, _): + alternate = PROFILE.model_copy(update={"profile_id": "alternate"}) + initial = declaration() + declared = ArtifactSearchDeclaration( + version="1", + indexes=( + *initial.indexes, + initial.indexes[1].model_copy(update={"name": "alternate", "profile": alternate}), + ), + modes=initial.modes, + default_mode="text", + defaults={"vector": {"vector": ("visual",)}, "hybrid": {"text": ("lexical",), "vector": ("visual",)}}, + ) + await register(db, "reports", declared) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + for selected, profile in (("visual", PROFILE), ("alternate", alternate)): + request = { + "mode": "vector", + "indexes": {"vector": [selected]}, + "query_vector": {"profile": profile.model_dump(mode="json"), "vector": [0, 0]}, + } + assert not (await search.search("scope", "reports", request)).matches + other_profile = alternate if selected == "visual" else PROFILE + request["query_vector"]["profile"] = other_profile.model_dump(mode="json") + with pytest.raises(ArtifactSearchUnsupported): + await search.search("scope", "reports", request) + + asyncio.run(run()) + + +def test_named_indexes_share_build_and_space_without_cross_matching(): + async def run(): + async with store() as (db, builder, publisher): + search_config = ArtifactSearchDeclaration( + version="1", + indexes=( + TextIndex(name="title", fields=("title",), projection_version="1"), + TextIndex(name="body", fields=("text",), projection_version="1"), + VectorIndex( + name="near", + source="content", + vector_field="v1", + profile_field="profile", + profile=PROFILE, + projection_version="1", + ), + VectorIndex( + name="far", + source="content", + vector_field="v2", + profile_field="profile", + profile=PROFILE, + projection_version="1", + ), + ), + modes=("text", "vector"), + default_mode="text", + defaults={"text": {"text": ("title",)}, "vector": {"vector": ("near",)}}, + ) + await register(db, "reports", search_config) + await write( + db, + publisher, + "scope", + "reports", + "a", + { + "title": "标题", + "text": "正文", + "v1": [0, 0], + "v2": [10, 0], + "profile": PROFILE.model_dump(mode="json"), + }, + ) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + for name, query, expected in (("title", "正文", False), ("body", "正文", True)): + assert ( + bool( + (await search.search("scope", "reports", {"query": query, "indexes": {"text": [name]}})).matches + ) + is expected + ) + for name, distance in (("near", 0), ("far", 10)): + found = await search.search( + "scope", + "reports", + { + "mode": "vector", + "indexes": {"vector": [name]}, + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]}, + "include_scores": True, + }, + ) + assert found.matches[0].channel_scores["vector"].raw == distance + + asyncio.run(run()) + + +def test_inflight_content_preparation_obeys_current_build_state(): + async def run(): + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + original = await write(db, publisher, "scope", "reports", "a", body("before", [0, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + content = body("recovered", [1, 0]) + prepared = await publisher.prepare("scope", "reports", "1", content) + await builder.refresh("scope", "reports", rebuild=True) + # Preparation started when ready; commit now uses only the change log. + repository = custom_artifact_repository("reports", prepared={"a": prepared}) + draft = RepositoryArtifactDraft( + family="reports", content=StoredArtifactContent(schema_version="1", content=content) + ) + async with db.transaction() as connection: + revised = await repository.revise( + connection, "scope", await repository.get(connection, "scope", original.as_ref()), draft + ) + pending = await publisher.prepare("scope", "reports", "1", body("too-late", [2, 0])) + assert await builder.build("scope", "reports") + repository = custom_artifact_repository("reports", prepared={"a": pending}) + with pytest.raises(ValueError, match="prepare again"): + async with db.transaction() as connection: + await repository.revise( + connection, + "scope", + revised, + RepositoryArtifactDraft( + family="reports", + content=StoredArtifactContent(schema_version="1", content=body("too-late", [2, 0])), + ), + ) + assert (await service(db, "reports").search("scope", "reports", {"query": "recovered"})).artifacts[ + 0 + ].revision == 2 + + asyncio.run(run()) + + +def test_superseded_build_cannot_write_clean_or_publish_and_cleanup_resumes(): + async def run(): + from powercontext.builtin.persistence.artifact_indexes import COVERAGE, TEXTS, entries + + async with store() as (db, builder, publisher): + await register(db, "reports", declaration()) + for scope, family in (("scope", "reports"), ("other-scope", "reports"), ("scope", "other-family")): + if family != "reports": + await register(db, family, declaration()) + await write(db, publisher, scope, family, "a", body("保留", [1, 0])) + await builder.refresh(scope, family) + assert await builder.build(scope, family) + await builder.refresh("scope", "reports", rebuild=True) + async with db.transaction() as connection: + old = ( + ( + await connection.execute( + select(STATES).where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + ) + ) + .mappings() + .one() + ) + assert not await builder.clean("scope", "reports", old["build_id"], limit=1) + # Restart repair with only part of the old rows removed. + await builder.refresh("scope", "reports", rebuild=True) + assert not await builder.clean("scope", "reports", old["build_id"]) + definition = entries(old["manifest"])[0][1]["index_def_id"] + assert not await builder._project_current("scope", "reports", old["build_id"], definition, "a") + assert not await builder._validate_and_publish("scope", "reports", old["build_id"]) + restarted = ArtifactIndexBuilder(db) + assert await restarted.build("scope", "reports") + for scope, family in (("scope", "reports"), ("other-scope", "reports"), ("scope", "other-family")): + assert (await service(db, family).search(scope, family, {"query": "保留"})).matches + async with db.transaction() as connection: + current = ( + ( + await connection.execute( + select(STATES).where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + ) + ) + .mappings() + .one() + ) + assert current["cleanup_targets"] == [] + for table in (TEXTS, COVERAGE): + assert not ( + await connection.execute( + select(table).where( + table.c.scope_id == "scope", + table.c.family == "reports", + table.c.build_id != current["build_id"], + ) + ) + ).all() + # A crashed final validator must not leave the build permanently occupied. + await restarted.refresh("scope", "reports", rebuild=True) + async with db.transaction() as connection: + await connection.execute( + update(STATES) + .where(STATES.c.scope_id == "scope", STATES.c.family == "reports") + .values(validation_token="interrupted") # noqa: S106 - Persisted build fencing token, not a credential. + ) + assert await ArtifactIndexBuilder(db).build("scope", "reports") + + asyncio.run(run()) + + +def test_repair_restart_rejects_an_inflight_encoder_and_keeps_new_build_unavailable(): + async def run(): + entered, release = asyncio.Event(), asyncio.Event() + + class Encoder: + profile = PROFILE + + async def embed(self, texts): + entered.set() + await release.wait() + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + async with store(model=Encoder()) as (db, builder, publisher): + config = ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", fields=("text",), source="text", profile=PROFILE, projection_version="1" + ), + ), + modes=("vector",), + default_mode="vector", + ) + await register(db, "reports", config) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库"}) + await builder.refresh("scope", "reports") + old_task = asyncio.create_task(builder.build("scope", "reports")) + try: + await asyncio.wait_for(entered.wait(), 5) + await builder.refresh("scope", "reports", rebuild=True) + release.set() + assert not await old_task + request = {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}} + with pytest.raises(ArtifactSearchUnavailable): + await service(db, "reports").search("scope", "reports", request) + assert await builder.build("scope", "reports") + assert (await service(db, "reports").search("scope", "reports", request)).matches + finally: + release.set() + await old_task + + asyncio.run(run()) + + +@pytest.mark.parametrize("during_build", [False, True]) +def test_query_policy_update_preserves_build_and_does_not_reencode(during_build): + async def run(): + entered, release = asyncio.Event(), asyncio.Event() + + class Encoder: + profile = PROFILE + encoded = False + + async def embed(self, texts): + assert not self.encoded, "query policy must not re-encode stored content" + self.encoded = True + entered.set() + await release.wait() + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + async with store(model=Encoder()) as (db, builder, publisher): + initial = ArtifactSearchDeclaration( + version="1", + indexes=( + TextIndex(name="title", fields=("title",), projection_version="1"), + TextIndex(name="body", fields=("text",), projection_version="1"), + VectorIndex( + name="semantic", fields=("text",), source="text", profile=PROFILE, projection_version="1" + ), + ), + modes=("text", "vector"), + default_mode="text", + defaults={"text": {"text": ("title",)}, "vector": {"vector": ("semantic",)}}, + ) + await register(db, "reports", initial) + await write(db, publisher, "scope", "reports", "a", {"title": "标题", "text": "正文"}) + await builder.refresh("scope", "reports") + build = asyncio.create_task(builder.build("scope", "reports")) + try: + async with asyncio.timeout(5): + await entered.wait() + if not during_build: + release.set() + assert await build + async with db.transaction() as connection: + before = dict((await connection.execute(select(STATES))).mappings().one()) + changed = initial.model_copy( + update={ + "default_mode": "vector", + "defaults": {"text": {"text": ("body",)}, "vector": {"vector": ("semantic",)}}, + "candidate_limit": 17, + "rrf_k": 11, + "text_weight": 2, + "vector_weight": 3, + } + ) + await register(db, "reports", changed) + await builder.refresh("scope", "reports") + async with db.transaction() as connection: + after = dict((await connection.execute(select(STATES))).mappings().one()) + assert after == before # The build identity, progress and readiness must survive policy changes. + release.set() + assert await build + assert await builder.build("scope", "reports") + search = service(db, "reports") + result = await search.search( + "scope", + "reports", + { + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + }, + ) + assert [item.artifact_id for item in result.artifacts] == ["a"] + assert (await search.search("scope", "reports", {"mode": "text", "query": "正文"})).artifacts + assert not (await search.search("scope", "reports", {"mode": "text", "query": "标题"})).artifacts + finally: + release.set() + await build + + asyncio.run(run()) + + +def test_inflight_query_keeps_policy_while_next_query_uses_update(): + async def run(): + entered, release = asyncio.Event(), asyncio.Event() + + class QueryEncoder: + profile = PROFILE + + async def embed(self, texts): + entered.set() + await release.wait() + return EmbeddingResult(vectors=((1.0, 0.0),)) + + async with store() as (db, builder, publisher): + initial = declaration().model_copy(update={"default_mode": "vector"}) + await register(db, "reports", initial) + await write(db, publisher, "scope", "reports", "a", body("数据库", [1, 0])) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports", QueryEncoder()) + request = {"query": "no lexical match"} + pending = asyncio.create_task(search.search("scope", "reports", request)) + try: + async with asyncio.timeout(5): + await entered.wait() + await register(db, "reports", initial.model_copy(update={"modes": ("text",), "default_mode": "text"})) + release.set() + assert [item.artifact_id for item in (await pending).artifacts] == ["a"] + assert not (await search.search("scope", "reports", request)).artifacts + finally: + release.set() + await pending + + asyncio.run(run()) diff --git a/tests/builtin/persistence/test_artifact_storage_lifecycle.py b/tests/builtin/persistence/test_artifact_storage_lifecycle.py new file mode 100644 index 0000000000..133afa3075 --- /dev/null +++ b/tests/builtin/persistence/test_artifact_storage_lifecycle.py @@ -0,0 +1,362 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fixed revisions preserve Artifact declarations and online native spaces.""" + +import asyncio +import json +import shutil + +import pytest +from sqlalchemy import insert, select, update +from sqlalchemy.schema import CreateTable + +from powercontext.builtin.persistence.artifact_indexes import ( + SCHEMAS, + STATES, + ArtifactIndexBuilder, + ArtifactProjectionPublisher, + register_definition, +) +from powercontext.builtin.persistence.artifact_vector_space import ( + native_vector_table, + prepare_family_spaces, + verify_native_space, +) +from powercontext.builtin.persistence.migrations.bundle import MigrationBundle +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner +from powercontext.builtin.persistence.migrations.mysql_schema import _parse_table, mysql_inventory +from powercontext.builtin.persistence.migrations.server_readiness import check_server_schema +from powercontext.builtin.persistence.tables import ARTIFACT_VECTOR_INDEXES_TABLE as DEFINITIONS +from powercontext.builtin.persistence.tables import ARTIFACT_VECTOR_SPACES_TABLE as SPACES +from tests.builtin.persistence.test_artifact_indexes import PROFILE, body, declaration, service, write +from tests.builtin.persistence.test_mysql_migration_runner import ( + _apply, + _full_server_runner, + _interrupt_after_ddl, + backend, # noqa: F401 - Real disposable backend fixture. +) +from tests.e2e.atomic_memory_migration_backend import migration_profile + + +def test_fixed_artifact_storage_resumes_after_registration_columns(backend): # noqa: F811 + runner = _full_server_runner(backend) + _interrupt_after_ddl( + runner, + "ALTER TABLE pc_artifact_processing_binding_states ADD COLUMN plugin_id", + policy="auto", + ) + (journal,) = backend.evidence_directory.rglob("maintenance.json") + interrupted = json.loads(journal.read_text()) + assert interrupted["last_revision"] == "v1_4_1_006" + assert interrupted["pending_revision"] == "v1_5_0_001" + recovery = _full_server_runner(backend).plan() + assert recovery.state == "recovery_required" and recovery.source_revision == "v1_4_1_006" + # The DDL returned and its owned connection closed before the injected failure. + resumed = _full_server_runner(backend, acknowledge_previous_run=(recovery.coordination or {}).get("pending_run_id")) + result = _apply(resumed) + assert result.state == "ready" and result.revision == resumed.bundle.head + assert resumed.verify().state == "ready" + completed = json.loads(journal.read_text()) + assert completed["state"] == "complete" + assert completed["maintenance_window_id"] == interrupted["maintenance_window_id"] + assert completed["source_fingerprint"] == interrupted["source_fingerprint"] + + +def _next_fixed_revision(bundle, directory): + """A subsequent fixed-table addition leaves the space format untouched.""" + shutil.copytree(bundle.directory, directory) + revision = "v1_5_0_002" + sql = "CREATE TABLE pc_lifecycle_probe (id INTEGER NOT NULL, PRIMARY KEY (id))" + name, layout = _parse_table(sql) + resource = "schemas/mysql-lifecycle-probe.json" + (directory / resource).write_text( + json.dumps({ + "format": "powercontext.mysql-state-layouts.v1", + "revisions": {revision: {"statements": [sql], "changes": [{name: layout}]}}, + }) + ) + (directory / "versions/v1_5_0_002_probe.py").write_text( + f"revision = '{revision}'\ndown_revision = '{bundle.head}'\ndef upgrade():\n pass\n" + ) + (directory / "mysql/versions/v1_5_0_002.py").write_text( + "from alembic import context\n" + f"revision = '{revision}'\ndown_revision = '{bundle.head}'\n" + "def upgrade():\n context.config.attributes['execute_revision'](revision)\n" + ) + path = directory / "manifest.json" + manifest = json.loads(path.read_text()) + manifest["revision_resources"][revision] = [resource] + manifest["mysql_state_layouts"].append(resource) + manifest["sqlite_fingerprints"][revision] = manifest["sqlite_fingerprints"][bundle.head] + manifest["domain_readiness"][revision] = manifest["domain_readiness"][bundle.head] + manifest["affected_objects"].append(name) + path.write_text(json.dumps(manifest)) + return MigrationBundle(directory) + + +def test_native_space_online_recovery_and_fixed_upgrade(backend, tmp_path): # noqa: F811 + runner = _full_server_runner(backend) + _apply(runner) + search = declaration() + vector = search.indexes[1] + + async def exercise(): + async with migration_profile(backend.config) as profile: + db = profile.database + async with db.transaction() as connection: + await connection.execute( + insert(SCHEMAS).values( + family="lifecycle", + schema_version="1", + declaration={ + "family": "lifecycle", + "schema_version": "1", + "search": search.model_dump(mode="json"), + }, + ) + ) + await register_definition(connection, "lifecycle", "1", vector) + async with db.transaction() as connection: + space = dict( + ( + await connection.execute( + select(SPACES).where( + SPACES.c.space_id.in_( + select(DEFINITIONS.c.space_id).where(DEFINITIONS.c.family == "lifecycle") + ) + ) + ) + ) + .mappings() + .one() + ) + assert space["provision_state"] == "declared" + await connection.run_sync(lambda c: check_server_schema(c, _startup=True)) + with pytest.raises(MigrationError): + await asyncio.to_thread(runner.plan, backup_policy="manual") + # Interrupt after CREATE TABLE, before the standalone index or ready. + async with db.transaction() as connection: + await connection.run_sync(lambda c: c.execute(CreateTable(native_vector_table(space)))) + await asyncio.gather(prepare_family_spaces(db, "lifecycle"), prepare_family_spaces(db, "lifecycle")) + async with db.transaction() as connection: + ready = dict( + ( + await connection.execute( + select(SPACES).where( + SPACES.c.space_id.in_( + select(DEFINITIONS.c.space_id).where(DEFINITIONS.c.family == "lifecycle") + ) + ) + ) + ) + .mappings() + .one() + ) + assert ready["space_id"] == space["space_id"] and ready["provision_state"] == "ready" + assert (await connection.execute(select(STATES))).first() is None + await connection.run_sync(lambda c: verify_native_space(c, ready)) + await register_definition(connection, "lifecycle", "1", vector) + # A table-created-before-ready interruption also resumes without DDL. + async with db.transaction() as connection: + await connection.execute( + update(SPACES).where(SPACES.c.space_id == ready["space_id"]).values(provision_state="declared") + ) + await prepare_family_spaces(db, "lifecycle") + publisher = ArtifactProjectionPublisher(db) + await write(db, publisher, "scope", "lifecycle", "preserved", body("retained", [1, 0])) + builder = ArtifactIndexBuilder(db) + await builder.refresh("scope", "lifecycle") + assert await builder.build("scope", "lifecycle") + found = await service(db, "lifecycle").search( + "scope", + "lifecycle", + { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + }, + ) + assert [item.artifact_id for item in found.artifacts] == ["preserved"] + return ready + + async def server_registration(): + import httpx + from pydantic import SecretStr + + from powercontext.builtin.runtime.config import RuntimeConfig + from powercontext.server.factory import create_server_app + from powercontext.server.settings import ( + BearerAuthConfig, + DashboardConfig, + McpConfig, + MetricsConfig, + ServerSettings, + ) + from tests.e2e.test_artifact_plugins import ADMIN_AUTHORIZATION_VALUE, EXAMPLE, deploy_plugin, reload_plugin + from tests.e2e.test_artifact_vector_http import _DECLARATION + + settings = ServerSettings( + database=backend.config, + workspace=tmp_path, + artifact_plugins_dir=tmp_path / "plugins", + runtime=RuntimeConfig(artifact_processing_families=()), + auth=BearerAuthConfig(enabled=True, token=SecretStr(ADMIN_AUTHORIZATION_VALUE)), + dashboard=DashboardConfig(enabled=False), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + app = create_server_app(settings=settings, scheduler_path=tmp_path / "scheduler.db") + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + headers={"Authorization": f"Bearer {ADMIN_AUTHORIZATION_VALUE}"}, + ) as client, + migration_profile(backend.config) as profile, + ): + plugin = deploy_plugin(EXAMPLE, tmp_path / "http-plugin", None) + entry = plugin / "plugin.py" + entry.write_text(entry.read_text() + _DECLARATION.replace("features-v1", "online-http-profile")) + await reload_plugin(client, tmp_path, plugin, 0) + async with asyncio.timeout(30): + while True: + async with profile.database.transaction() as connection: + spaces = (await connection.execute(select(SPACES))).mappings().all() + if any( + row["profile_json"]["profile_id"] == "online-http-profile" + and row["provision_state"] == "ready" + for row in spaces + ): + assert ( + await connection.execute(select(STATES).where(STATES.c.family == "incident-report")) + ).first() is None + break + await asyncio.sleep(0.1) + + asyncio.run(server_registration()) + ready = asyncio.run(exercise()) + next_bundle = _next_fixed_revision(runner.bundle, tmp_path / "next") + upgrade = MySQLMigrationRunner( + backend.connections(), + next_bundle, + evidence_directory=backend.evidence_directory, + configuration=runner.configuration, + ) + plan = upgrade.plan(backup_policy="manual") + assert ready["physical_table"] in plan.affected_objects + + async def concurrent_registration(): + async with migration_profile(backend.config) as profile: + db = profile.database + other_search = declaration(profile=PROFILE.model_copy(update={"profile_id": "later-profile"})) + + async def register(family): + async with db.transaction() as connection: + await connection.execute( + insert(SCHEMAS).values( + family=family, + schema_version="1", + declaration={ + "family": family, + "schema_version": "1", + "search": other_search.model_dump(mode="json"), + }, + ) + ) + await register_definition(connection, family, "1", other_search.indexes[1]) + + await asyncio.gather(register("other-a"), register("other-b")) + await asyncio.gather(prepare_family_spaces(db, "other-a"), prepare_family_spaces(db, "other-b")) + async with db.transaction() as connection: + assert len((await connection.execute(select(SPACES))).all()) == 3 + + asyncio.run(concurrent_registration()) + with pytest.raises(MigrationError, match="plan_changed"): + upgrade.apply( + plan_id=plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="manual", + backup_confirmed=True, + ) + before = backend.connections().run(lambda c, _i, _g: mysql_inventory(c)) + result = _apply(upgrade, policy="manual") + assert result.changed and result.backup_state == "user_confirmed" + after = backend.connections().run(lambda c, _i, _g: mysql_inventory(c)) + assert before["artifact_spaces"] == after["artifact_spaces"] + assert before["tables"][ready["physical_table"]] == after["tables"][ready["physical_table"]] + + # Ready damage is rejected, never recreated by online preparation. + async def damage(): + async with migration_profile(backend.config) as profile: + found = await service(profile.database, "lifecycle").search( + "scope", + "lifecycle", + { + "mode": "vector", + "query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}, + }, + ) + assert [item.artifact_id for item in found.artifacts] == ["preserved"] + async with profile.database.transaction() as connection: + await connection.exec_driver_sql(f"DROP TABLE `{ready['physical_table']}`") + with pytest.raises(ValueError, match="absent"): + await prepare_family_spaces(profile.database, "lifecycle") + + asyncio.run(damage()) + with pytest.raises(MigrationError): + upgrade.plan(backup_policy="manual") + + +def test_managed_readiness_requires_durable_plugin_binding(tmp_path): + from sqlalchemy import create_engine, text + + from tests.test_managed_maintenance_evidence import _managed + + path, _settings, _runner = _managed(tmp_path, None) + engine = create_engine(f"sqlite:///{path}") + try: + with engine.begin() as connection: + connection.execute( + insert(SCHEMAS).values( + family="lifecycle", + schema_version="1", + declaration={"family": "lifecycle", "schema_version": "1"}, + ) + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_processing_binding_states " + "(binding_name,plugin_id,plugin_family,desired_state,phase) " + "VALUES ('custom-handler','plugin','lifecycle','unloaded','unloaded')" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_processing_intents " + "(binding_name,scope_id,pending_sequence,dirty_generation,clean_generation,requested_generation," + "handled_generation,last_auto_scan_generation) VALUES ('custom-handler','scope',1,0,0,1,0,0)" + ) + with engine.connect() as connection: + check_server_schema(connection, _startup=True) + with engine.begin() as connection: + connection.execute( + text( + "UPDATE pc_artifact_processing_binding_states SET plugin_family='undeclared' " + "WHERE binding_name='custom-handler'" + ) + ) + with engine.connect() as connection, pytest.raises(MigrationError, match="unsupported_task_format"): + check_server_schema(connection, _startup=True) + finally: + engine.dispose() diff --git a/tests/builtin/persistence/test_atomic_description_vector_layout.py b/tests/builtin/persistence/test_atomic_description_vector_layout.py index abf1fba78f..1c41ce29e4 100644 --- a/tests/builtin/persistence/test_atomic_description_vector_layout.py +++ b/tests/builtin/persistence/test_atomic_description_vector_layout.py @@ -42,7 +42,7 @@ def test_managed_sqlite_vector_schema_includes_registered_nullable_description(t database = tmp_path / "vector.db" runner = SQLiteMigrationRunner(database, bundle, configuration=configuration) result = runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) - assert result.revision == "v1_4_1_006" + assert result.revision == "v1_5_0_001" with closing(sqlite3.connect(database)) as connection: columns = {row[1]: row for row in connection.execute("PRAGMA table_info(pc_atomic_memory_current)")} assert columns["description"][2:4] == ("VARCHAR(200)", 0) @@ -56,6 +56,8 @@ def test_mysql_vector_admission_retains_description_and_exact_previous_revision( assert vector.vector_profile is not None inventory = {"tables": {}, "views": [], "triggers": []} for resource in json.loads((plain.directory / "manifest.json").read_text())["mysql_state_layouts"]: + if resource.endswith("mysql-v1_5_0_001.json"): + continue for entry in json.loads((plain.directory / resource).read_text())["revisions"].values(): for change in entry["changes"]: for name, table in change.items(): @@ -101,19 +103,19 @@ def test_description_upgrade_preserves_registered_owner_recovery_and_evidence(tm shutil.copytree(current.snapshot_directory, directory, ignore=shutil.ignore_patterns("__pycache__")) manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - plugin_resources = manifest["revision_resources"].pop("v1_4_1_006") - for resource in (*plugin_resources, "versions/v1_4_1_006_source_write_plugins.py", "mysql/versions/v1_4_1_006.py"): - (directory / resource).unlink() - del manifest["sqlite_fingerprints"]["v1_4_1_006"] - del manifest["domain_readiness"]["v1_4_1_006"] - manifest["mysql_state_layouts"].remove("schemas/mysql-v1_4_1_006.json") + removed_revisions = ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + for revision in removed_revisions: + for resource in manifest["revision_resources"].pop(revision): + (directory / resource).unlink() + for script in (directory / "versions").glob(f"{revision}_*.py"): + script.unlink() + (directory / "mysql/versions" / f"{revision}.py").unlink() + del manifest["sqlite_fingerprints"][revision] + del manifest["domain_readiness"][revision] + manifest["mysql_state_layouts"].remove(f"schemas/mysql-{revision}.json") + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in removed_revisions] + (directory / "artifact_storage_readiness.py").unlink() manifest["affected_objects"].remove("pc_source_write_plugins") - resources = manifest["revision_resources"].pop("v1_4_1_005") - for resource in (*resources, "versions/v1_4_1_005_atomic_memory_description.py", "mysql/versions/v1_4_1_005.py"): - (directory / resource).unlink() - del manifest["sqlite_fingerprints"]["v1_4_1_005"] - del manifest["domain_readiness"]["v1_4_1_005"] - manifest["mysql_state_layouts"].remove("schemas/mysql-v1_4_1_005.json") manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") previous = MigrationBundle(directory, embedding_profile=profile) configuration = { @@ -131,8 +133,8 @@ def test_description_upgrade_preserves_registered_owner_recovery_and_evidence(tm original_evidence = json.loads(old.evidence_path.read_text(encoding="utf-8")) upgraded = SQLiteMigrationRunner(database, current, configuration=configuration) plan = upgraded.plan() - assert plan.source_revision == "v1_4_1_004" and plan.revisions == ("v1_4_1_005", "v1_4_1_006") - assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_4_1_006" + assert plan.source_revision == "v1_4_1_004" and plan.revisions == removed_revisions + assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_5_0_001" assert upgraded.verify().state == "ready" evidence = json.loads(upgraded.evidence_path.read_text(encoding="utf-8")) for revision, checksum in original_evidence["release_checksums"].items(): diff --git a/tests/builtin/persistence/test_custom_artifact_search.py b/tests/builtin/persistence/test_custom_artifact_search.py new file mode 100644 index 0000000000..8294ab9709 --- /dev/null +++ b/tests/builtin/persistence/test_custom_artifact_search.py @@ -0,0 +1,121 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Initialize missing search tables in existing Artifact storage, preserving exact historical content.""" + +import asyncio +import sqlite3 + +from sqlalchemy import insert + +from powercontext.artifacts import Artifact +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.artifacts import ArtifactRepository, RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_SEARCH_TABLES, + BUILTIN_TABLES, +) +from tests.builtin.persistence.test_artifact_indexes import service + + +def test_existing_database_adds_search_tables_and_rebuilds_legacy_declarations(tmp_path): + path = tmp_path / "existing.db" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{path}") + + async def legacy(): + tables = tuple(table for table in BUILTIN_TABLES if table not in ARTIFACT_SEARCH_TABLES) + async with SQLiteProfile.open(config, tables=tables) as profile: + artifact_type = type("Legacy", (Artifact[StoredArtifactContent],), {"family": "reports"}) + repository = ArtifactRepository((artifact_type,)) + async with profile.database.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family="reports", + schema_version="1", + declaration={"family": "reports", "schema_version": "1", "search_fields": ["body"]}, + ) + ) + first = await repository.create( + connection, + "scope", + "a", + RepositoryArtifactDraft( + family="reports", + content=StoredArtifactContent(schema_version="1", content={"body": "数据库 TIMEOUT"}), + ), + ) + await repository.revise( + connection, + "scope", + first, + RepositoryArtifactDraft( + family="reports", + content=StoredArtifactContent(schema_version="1", content={"body": "连接恢复 RECOVERED"}), + ), + ) + + asyncio.run(legacy()) + with sqlite3.connect(path) as connection: + before = connection.execute("SELECT * FROM pc_artifacts ORDER BY revision").fetchall() + assert not connection.execute("SELECT 1 FROM sqlite_master WHERE name='pc_artifact_search_states'").fetchall() + + async def rebuilt(): + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as profile: + builder = ArtifactIndexBuilder(profile.database) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(profile.database, "reports") + assert not (await search.search("scope", "reports", {"query": "TIMEOUT"})).matches + result = await search.search("scope", "reports", {"query": "RECOVERED"}) + assert result.artifacts[0].revision == 2 + assert result.artifacts[0].content.model_dump() == {"body": "连接恢复 RECOVERED"} + + asyncio.run(rebuilt()) + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT * FROM pc_artifacts ORDER BY revision").fetchall() == before + assert not connection.execute("SELECT 1 FROM sqlite_master WHERE name='pc_artifact_search_schema'").fetchall() + connection.execute("DROP TRIGGER pc_artifact_search_fts_update") + # Missing full-text support is recreated; repeated initialization preserves exact content. + asyncio.run(rebuilt()) + + +def test_intermediate_search_schema_is_rejected_without_deleting_existing_rows(tmp_path): + import pytest + + path = tmp_path / "incompatible.db" + + async def initialize_database(): + tables = tuple(table for table in BUILTIN_TABLES if table not in ARTIFACT_SEARCH_TABLES) + async with SQLiteProfile.open(SQLiteConfig(url=f"sqlite+aiosqlite:///{path}"), tables=tables): + pass + + asyncio.run(initialize_database()) + with sqlite3.connect(path) as connection: + connection.execute( + "CREATE TABLE pc_artifact_search_states (scope_id TEXT, family TEXT, release_id INTEGER, serving_manifest TEXT, desired_manifest TEXT, change_seq INTEGER, PRIMARY KEY(scope_id, family))" + ) + connection.execute("INSERT INTO pc_artifact_search_states VALUES ('scope', 'reports', 3, '{}', '{}', 9)") + before = connection.execute("SELECT * FROM pc_artifact_search_states").fetchall() + + async def open_old_database(): + with pytest.raises(ValueError, match=r"incompatible.*migration"): + async with SQLiteProfile.open(SQLiteConfig(url=f"sqlite+aiosqlite:///{path}"), tables=BUILTIN_TABLES): + pytest.fail("Old search layout must not be treated as upgraded") + + asyncio.run(open_old_database()) + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT * FROM pc_artifact_search_states").fetchall() == before diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index 66ed8646bc..3f74335524 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -73,8 +73,8 @@ def dump(path): def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): before = dump(source.database) plan = source.plan() - assert plan.source_revision == "s1300" and plan.target_revision == "v1_4_1_006" - assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006") + assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_001" + assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") assert dump(source.database) == before result = apply(source) assert result.readiness_scope == "server" and result.backup_state == "completed" diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 0357bbcf43..f50ab3d23b 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -561,7 +561,7 @@ def test_oceanbase_server_upgrade_with_separate_code_cache_preserves_history( ) assert runner.plan(backup_policy="manual").source_revision == "s1300" result = _apply(runner, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_4_1_006" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_001" assert runner.verify().state == "ready" prepare_server_storage(settings) assert not _apply(runner, policy="manual").changed @@ -692,7 +692,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_4_1_006", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_001", "dedicated", b'{"history":"unchanged"}') (main_path,) = backend.evidence_directory.glob("*/maintenance.json") evidence = json.loads(main_path.read_text()) @@ -702,7 +702,7 @@ def inspect_history(connection, identity, guard): for inspect in (runner.plan, runner.verify, runner.apply): with pytest.raises(MigrationError, match="checksum_conflict"): inspect() - assert backend.connections().run(inspect_history) == ("v1_4_1_006", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_001", "dedicated", b'{"history":"unchanged"}') assert { path: path.read_bytes() for path in backend.evidence_directory.rglob("maintenance.json") } == original_evidence @@ -769,7 +769,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_4_1_006", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_001", b'{"history":"unchanged"}') def test_mysql_registered_projection_repair_preserves_authority_and_revision(backend: _BackendTarget) -> None: @@ -837,7 +837,7 @@ def inspect_content(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_content) == ("v1_4_1_006", content, "Preserve immutable nebula evidence.") + assert backend.connections().run(inspect_content) == ("v1_5_0_001", content, "Preserve immutable nebula evidence.") @pytest.mark.parametrize("origin", ("empty", "released")) @@ -992,7 +992,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan _load_full_server_baseline(backend) runner = _full_server_runner(backend) original = runner.plan(backup_policy="manual") - assert original.source_revision == "s1300" and original.target_revision == "v1_4_1_006" + assert original.source_revision == "s1300" and original.target_revision == "v1_5_0_001" _interrupt_after_ddl(runner, interruption) journals = list(backend.evidence_directory.rglob("maintenance.json")) assert len(journals) == 1 @@ -1014,7 +1014,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan result = _apply(resumed, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_4_1_006" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_001" assert result.backup_state == "user_confirmed" assert result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" @@ -1094,7 +1094,7 @@ def client_active(connection: Connection | None, _identity: BackendIdentity, _gu # interruption. An arbitrary remote failure cannot be acknowledged blindly. resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_4_1_006" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_001" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journals[0].read_text(encoding="utf-8")) diff --git a/tests/builtin/persistence/test_mysql_schema.py b/tests/builtin/persistence/test_mysql_schema.py index febd48735b..5f0b711062 100644 --- a/tests/builtin/persistence/test_mysql_schema.py +++ b/tests/builtin/persistence/test_mysql_schema.py @@ -195,7 +195,9 @@ def test_documented_obloader_restore_layers_are_parent_first() -> None: assert len(set(restore_plans)) == 1 restore_layers = restore_plans[0] - _assert_restore_layers_are_parent_first(restore_layers, BUILTIN_TABLES) + _assert_restore_layers_are_parent_first( + restore_layers, tuple(table for table in BUILTIN_TABLES if table.name != "pc_artifact_vectors") + ) def test_every_mysql_utf8mb4_key_stays_below_the_innodb_limit() -> None: diff --git a/tests/builtin/persistence/test_owner_recovery_migration.py b/tests/builtin/persistence/test_owner_recovery_migration.py index 24c12480d8..de62a3fa0c 100644 --- a/tests/builtin/persistence/test_owner_recovery_migration.py +++ b/tests/builtin/persistence/test_owner_recovery_migration.py @@ -15,38 +15,77 @@ """Append owner recovery storage without changing historical migration evidence.""" import json +import shutil import sqlite3 from contextlib import closing import pytest from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.migrations.bundle import MigrationBundle from powercontext.builtin.persistence.migrations.deployment import server_bundle from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner from tests.builtin.persistence.test_mysql_migration_schema import _inventory, _ReflectedDatabase -from tests.builtin.persistence.test_portable_storage_migration import apply, configuration, previous_bundle +from tests.builtin.persistence.test_portable_storage_migration import apply, configuration @pytest.mark.parametrize("vectors", [False, True]) -def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checksums(tmp_path, vectors): +@pytest.mark.parametrize("previous_revision", ["s1402", "v1_4_1_004", "v1_4_1_006"]) +def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checksums( + tmp_path, vectors, previous_revision +): embedding = EmbeddingProfile(profile_id="owner-vectors", model="fixture", dimension=3) if vectors else None current = server_bundle(embedding_profile=embedding) - previous = previous_bundle(tmp_path / "previous", embedding, include_portable=True) + directory = tmp_path / "previous" + shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) + manifest_path = directory / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + removed_revisions = ( + ("v1_5_0_001",) + if previous_revision == "v1_4_1_006" + else ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + if previous_revision == "v1_4_1_004" + else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + ) + for revision in removed_revisions: + for resource in manifest["revision_resources"].pop(revision): + (directory / resource).unlink() + for script in (directory / "versions").glob(f"{revision}_*.py"): + script.unlink() + (directory / "mysql/versions" / f"{revision}.py").unlink() + del manifest["sqlite_fingerprints"][revision] + del manifest["domain_readiness"][revision] + manifest["mysql_state_layouts"].remove(f"schemas/mysql-{revision}.json") + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in removed_revisions] + (directory / "artifact_storage_readiness.py").unlink() + if previous_revision != "v1_4_1_006": + manifest["affected_objects"].remove("pc_source_write_plugins") + if previous_revision == "s1402": + manifest["affected_objects"].remove("pc_access_owner_recovery") + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + previous = MigrationBundle(directory, embedding_profile=embedding) database = tmp_path / "managed.db" old = SQLiteMigrationRunner(database, previous, configuration=configuration(embedding)) - assert apply(old).revision == "s1402" + assert apply(old).revision == previous_revision with closing(sqlite3.connect(database)) as connection, connection: connection.execute( "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " "VALUES ('original','title','summary','original','title','summary',1)" ) + if previous_revision == "v1_4_1_006": + connection.execute("INSERT INTO pc_source_write_plugins VALUES (?, ?)", ("tickets", b"retained definition")) + if previous_revision == "v1_4_1_004": + connection.execute( + "INSERT INTO pc_access_owner_recovery (operation_id,deployment_id,scope_id,completed,payload) " + "VALUES ('pending','deployment','original',0,'retained intent')" + ) original_evidence = json.loads(old.evidence_path.read_text(encoding="utf-8")) upgraded = SQLiteMigrationRunner(database, current, configuration=configuration(embedding)) plan = upgraded.plan() - assert plan.source_revision == "s1402" and plan.revisions == ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006") - assert apply(upgraded).revision == "v1_4_1_006" + assert plan.source_revision == previous_revision and plan.revisions == removed_revisions + assert apply(upgraded).revision == "v1_5_0_001" assert upgraded.verify().state == "ready" assert not apply(upgraded).changed evidence = json.loads(upgraded.evidence_path.read_text(encoding="utf-8")) @@ -56,7 +95,14 @@ def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checks assert connection.execute("SELECT scope_id,title,version FROM pc_scopes").fetchall() == [ ("original", "title", 1) ] - assert connection.execute("SELECT COUNT(*) FROM pc_access_owner_recovery").fetchone() == (0,) + expected_intents = [("pending", "retained intent", 0)] if previous_revision == "v1_4_1_004" else [] + assert ( + connection.execute("SELECT operation_id,payload,completed FROM pc_access_owner_recovery").fetchall() + == expected_intents + ) + expected_plugins = [("tickets", b"retained definition")] if previous_revision == "v1_4_1_006" else [] + assert connection.execute("SELECT * FROM pc_source_write_plugins").fetchall() == expected_plugins + assert connection.execute("SELECT COUNT(*) FROM pc_artifact_family_schemas").fetchone() == (0,) fresh = SQLiteMigrationRunner(tmp_path / "fresh.db", current, configuration=configuration(embedding)) apply(fresh) assert upgraded.plan().schema_fingerprint == fresh.plan().schema_fingerprint diff --git a/tests/builtin/persistence/test_portable_storage_migration.py b/tests/builtin/persistence/test_portable_storage_migration.py index 8306425d6a..3c286ca286 100644 --- a/tests/builtin/persistence/test_portable_storage_migration.py +++ b/tests/builtin/persistence/test_portable_storage_migration.py @@ -36,6 +36,7 @@ def previous_bundle(directory, embedding, *, include_portable=False, include_own manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text()) later_revisions = [ + ("v1_5_0_001", "artifact_storage", None), ("v1_4_1_006", "source_write_plugins", "pc_source_write_plugins"), ("v1_4_1_005", "atomic_memory_description", None), ] @@ -53,6 +54,8 @@ def previous_bundle(directory, embedding, *, include_portable=False, include_own manifest["mysql_state_layouts"].remove(f"schemas/mysql-{later}.json") if table is not None: manifest["affected_objects"].remove(table) + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != "v1_5_0_001"] + (directory / "artifact_storage_readiness.py").unlink() if not include_portable: removed = manifest["revision_resources"].pop("s1402") for resource in ( @@ -110,8 +113,8 @@ def test_registered_s1401_upgrade_preserves_completed_evidence_and_matches_fresh upgraded = SQLiteMigrationRunner(database, bundle, configuration=config) plan = upgraded.plan() assert plan.source_revision == "s1401" - assert plan.revisions == ("s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006") - assert apply(upgraded).revision == "v1_4_1_006" + assert plan.revisions == ("s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + assert apply(upgraded).revision == "v1_5_0_001" assert upgraded.verify().state == "ready" assert not apply(upgraded).changed fresh = SQLiteMigrationRunner(tmp_path / "fresh.db", bundle, configuration=config) @@ -170,9 +173,11 @@ def test_source_write_plugin_upgrade_preserves_data_and_registration(tmp_path, v bundle = server_bundle(embedding_profile=embedding) upgraded = SQLiteMigrationRunner(database, bundle, configuration=config) assert upgraded.plan().revisions == ( - ("v1_4_1_005", "v1_4_1_006") if include_owner else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006") + ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + if include_owner + else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") ) - assert apply(upgraded).revision == "v1_4_1_006" + assert apply(upgraded).revision == "v1_5_0_001" definition = b'{"configuration":{"id":"tickets","entry":"tickets"}}' with closing(sqlite3.connect(database)) as connection, connection: assert connection.execute("SELECT title FROM pc_scopes").fetchall() == [("preserved",)] diff --git a/tests/builtin/persistence/test_release140_server_migrations.py b/tests/builtin/persistence/test_release140_server_migrations.py index 1c5e500c4d..18e011c459 100644 --- a/tests/builtin/persistence/test_release140_server_migrations.py +++ b/tests/builtin/persistence/test_release140_server_migrations.py @@ -134,7 +134,7 @@ def test_registered_sources_choose_distinct_lineages(tmp_path, release, expected _settings, _embedding, runner = _source(tmp_path, release=release) plan = runner.plan() assert plan.source_revision == expected - assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006") + assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") @pytest.mark.parametrize(("vectors", "code"), [(False, False), (True, False), (False, True), (True, True)]) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 48bd9b8a51..107fa4efaa 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -26,9 +26,14 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.codec import dump_model +from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile -from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE +from powercontext.builtin.persistence.tables import ( + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, + SHARED_TABLES, + SOURCES_TABLE, +) from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.sources import CONTENT_SOURCE_ADAPTER, ContentCapture from powercontext.sources import ( @@ -100,6 +105,56 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_source_batches_notify_matching_artifact_plugins_only_for_new_inputs() -> None: + async def scenario() -> None: + intents = ArtifactProcessingIntentRepository() + async with ( + repository_profile() as (profile, repositories), + profile.database.transaction() as connection, + ): + await connection.execute( + insert(ARTIFACT_PROCESSING_BINDING_STATES_TABLE), + [ + { + "binding_name": name, + "plugin_spec": specification, + "desired_state": "disabled", + } + for name, specification in ( + ("all-scopes", {"manifest": {"scope_ids": None}}), + ("matching-scope", {"manifest": {"scope_ids": ["scope-a"]}}), + ("other-scope", {"manifest": {"scope_ids": ["scope-b"]}}), + ("unloaded", None), + ) + ], + ) + first, second, third = ( + NoteSource(name=f"note-{number}", materialization=SourceMaterialization.CAPTURED, body="Input") + for number in range(3) + ) + batch = (first, first, second) + created = await repositories.sources.add_many_with_status(connection, "scope-a", batch) + assert [is_new for _, is_new in created] == [True, False, True] + for name in ("all-scopes", "matching-scope"): + pending = await intents.load(connection, "scope-a", name) + assert pending is not None and pending.dirty_generation == 2 + + replay = await repositories.sources.add_many_with_status(connection, "scope-a", batch) + assert all(not is_new for _, is_new in replay) + for name in ("all-scopes", "matching-scope"): + pending = await intents.load(connection, "scope-a", name) + assert pending is not None and pending.dirty_generation == 2 + + await repositories.sources.add(connection, "scope-a", third) + for name in ("all-scopes", "matching-scope"): + pending = await intents.load(connection, "scope-a", name) + assert pending is not None and pending.dirty_generation == 3 + for name in ("other-scope", "unloaded"): + assert await intents.load(connection, "scope-a", name) is None + + asyncio.run(scenario()) + + def test_reusing_a_stable_source_identity_with_different_payload_is_a_conflict() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): diff --git a/tests/builtin/runtime/test_artifact_plugin_worker.py b/tests/builtin/runtime/test_artifact_plugin_worker.py new file mode 100644 index 0000000000..ad559cac4b --- /dev/null +++ b/tests/builtin/runtime/test_artifact_plugin_worker.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Plugin Workers preserve deployment inference identity when opening storage.""" + +import asyncio +import hashlib +import json + +import pytest +import yaml +from sqlalchemy import insert + +from powercontext.builtin.artifact_plugin_validation import _PluginManifest, describe_module, load_plugin +from powercontext.builtin.persistence.custom_artifacts import custom_artifact_repository +from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository +from powercontext.builtin.persistence.tables import ( + ARTIFACT_FAMILY_SCHEMAS_TABLE, + ARTIFACT_PROCESSING_BINDING_STATES_TABLE, +) +from powercontext.builtin.runtime.artifact_plugins import ArtifactPluginWorkerSpec, run_artifact_plugin_worker +from powercontext.builtin.runtime.composition import open_builtin_contexts +from powercontext.builtin.runtime.config import BuiltinConfig, InferenceConfig, SQLiteConfig +from powercontext.builtin.runtime.processing_contracts import ArtifactProcessingWorkAssignment +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, ContentSource +from powercontext.sources import SourceMaterialization +from tests.e2e.test_artifact_plugins import EXAMPLE, deploy_plugin + + +@pytest.mark.parametrize("configured_inference", [False, True]) +def test_plugin_worker_generates_with_deployment_inference(tmp_path, configured_inference): + directory = str(deploy_plugin(EXAMPLE, tmp_path / "plugin", None)) + manifest = _PluginManifest.model_validate(yaml.safe_load((tmp_path / "plugin/plugin.yml").read_text())) + definition = describe_module(load_plugin(directory, manifest), manifest, directory) + definition["directory"] = directory + declaration = definition["declaration"] + family, binding = declaration["family"], manifest.binding_name + # Only storage composition is used: this model is never called by the deterministic plugin. + inference = InferenceConfig(generation_model="openai:gpt-4o-mini") if configured_inference else None + config = BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'worker.db'}"), + inference=inference or InferenceConfig(), + ) + + async def prepare(): + async with open_builtin_contexts(config) as storage, storage.database.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration=declaration, + ) + ) + await connection.execute( + insert(ARTIFACT_PROCESSING_BINDING_STATES_TABLE).values( + binding_name=binding, + binding_generation=1, + phase="enabled", + ) + ) + source = await SourceRepository(BUILTIN_SOURCE_REGISTRY).add( + connection, + "scope", + ContentSource( + name="ticket", + materialization=SourceMaterialization.CAPTURED, + content="Example ticket", + ), + ) + lease = await ArtifactProcessingLeaseRepository().start_single_process_term(connection, "test-owner") + intent = await ArtifactProcessingIntentRepository().request(connection, "scope", binding) + assignment = ArtifactProcessingWorkAssignment( + binding_name=binding, + scope_id="scope", + artifact_family=family, + claimed_request_generation=intent.requested_generation, + fence=lease.fence("single-process"), + worker_id="test-worker", + ) + return assignment, source.ref + + assignment, source_ref = asyncio.run(prepare()) + run_artifact_plugin_worker( + ArtifactPluginWorkerSpec( + config.database, + binding, + 1, + definition, + inference_config=inference, + runtime_config=config.runtime, + ), + assignment, + ) + + async def verify(): + async with open_builtin_contexts(config) as storage, storage.database.transaction() as connection: + identity = json.dumps(source_ref.model_dump(mode="json"), sort_keys=True, separators=(",", ":")) + artifact_id = hashlib.sha256(identity.encode()).hexdigest() + report = await custom_artifact_repository(family).latest(connection, "scope", family, artifact_id) + assert report.content.content["evidence_text"] == "Example ticket" + assert report.lineage.sources == (source_ref,) + intent = await ArtifactProcessingIntentRepository().load(connection, "scope", binding) + assert intent is not None + assert intent.handled_generation == assignment.claimed_request_generation + + asyncio.run(verify()) diff --git a/tests/builtin/runtime/test_artifact_read_context.py b/tests/builtin/runtime/test_artifact_read_context.py new file mode 100644 index 0000000000..15cc39ac4f --- /dev/null +++ b/tests/builtin/runtime/test_artifact_read_context.py @@ -0,0 +1,482 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The shared read context preserves scope, family and caller permissions.""" + +import asyncio +from contextlib import AsyncExitStack +from datetime import UTC, datetime +from functools import partial +from typing import cast + +import pytest +from sqlalchemy import insert, select + +from powercontext.artifact_plugins import ArtifactDraft, GenerationContext +from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.search import ArtifactSearchUnsupported +from powercontext.builtin.artifacts.profile.models import Profile, ProfileContent, ProfileGeneration +from powercontext.builtin.artifacts.topic_memory.models import TopicMemory, TopicMemoryContent +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.artifacts import ArtifactRepository, RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import ( + StoredArtifactContent, + custom_artifact_repository, +) +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.sqlite.experience_index import SQLiteExperienceFTSIndex +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE, BUILTIN_TABLES, SOURCE_CURSORS_TABLE +from powercontext.builtin.records import BaseValueNotFoundError, InvalidBaseAccessRequestError +from powercontext.builtin.runtime.artifact_plugins import ArtifactPluginWorkerSpec, _commit_window +from powercontext.builtin.runtime.artifact_reads import ArtifactReadSession +from powercontext.builtin.runtime.processing_contracts import ArtifactProcessingWorkAssignment +from powercontext.builtin.runtime.processing_execution import ScopeInvocation +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY +from powercontext.server.authz import ( + AccessAuditContext, + AccessControlService, + AccessDecision, + AccessDeniedError, + AccessRole, + BuiltinAuthorizationProvider, + CreateBinding, + PrincipalRef, + ResourceRef, +) +from powercontext.server.authz.composition import open_builtin_access_control +from powercontext.server.authz.profiles import register_custom_artifact_access_family +from powercontext.server.authz.repository import RelationalAccessRepository +from powercontext.server.processing_security import WorkerSecurity + + +class FilteredProvider(BuiltinAuthorizationProvider): + async def check_batch(self, requests, /): + decisions = await super().check_batch(requests) + return tuple( + AccessDecision(False, "restricted-artifact", decision.policy_revision) + if request.resource.artifact_id == "a-hidden" + else decision + for request, decision in zip(requests, decisions, strict=True) + ) + + +def test_generation_reads_share_permission_filtered_candidates_and_exact_evidence(tmp_path): + async def scenario(): + admin = PrincipalRef(type="service", id="admin") + alice, bob = PrincipalRef(type="user", id="alice"), PrincipalRef(type="user", id="bob") + audit = AccessAuditContext(transport="test", operation="artifact-reads") + family = "context-items" + register_custom_artifact_access_family(family) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'reads.db'}") + async with AsyncExitStack() as resources: + profile = await resources.enter_async_context(SQLiteProfile.open(config, tables=BUILTIN_TABLES)) + access = await resources.enter_async_context( + open_builtin_access_control(config, bootstrap_administrators=(admin,)) + ) + await access.create_binding( + admin, + CreateBinding( + subject=alice, + resource=ResourceRef.scope("scope"), + role=AccessRole.SCOPE_VIEWER, + idempotency_key="scope-viewer", + ), + context=audit, + ) + repository = custom_artifact_repository(family) + async with profile.database.transaction() as connection: + await SQLiteExperienceFTSIndex().initialize(connection) + topic = await ArtifactRepository((TopicMemory,)).create( + connection, + "scope", + "topic", + RepositoryArtifactDraft( + family="topic-memory", + content=TopicMemoryContent(title="Topic", summary="Summary", detail="Evidence"), + ), + ) + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={"search_fields": ["text"], "schema_version": "1"}, + ) + ) + for scope, identity in (("scope", "a-hidden"), ("scope", "b-visible"), ("other", "foreign")): + await repository.create( + connection, + scope, + identity, + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent(schema_version="1", content={"text": "数据库 TIMEOUT"}), + ), + ) + builder = ArtifactIndexBuilder(profile.database) + await builder.refresh("scope", family) + assert await builder.build("scope", family) + for identity, owner in (("a-hidden", bob), ("b-visible", alice)): + await access.establish_artifact_owner( + ResourceRef.artifact("scope", family=family, artifact_id=identity), + owner, + idempotency_key=identity, + context=audit, + ) + builtin_reads = ArtifactReadSession( + profile.database, "scope", (family,), access=access, principal=alice, audit=audit + ) + builtin_page = await builtin_reads.list(family) + assert {ref.artifact_id for ref in builtin_page.items} == {"a-hidden", "b-visible"} + assert (await builtin_reads.read(family, "a-hidden", 1)).content["text"] == "数据库 TIMEOUT" + builtin_matches = await builtin_reads.search(family, "数据库", limit=2) + assert {hit.artifact_ref.artifact_id for hit in builtin_matches.matches} == {"a-hidden", "b-visible"} + # An injected policy may narrow object visibility; it is not the Worker provider. + access = AccessControlService( + FilteredProvider(cast(RelationalAccessRepository, access.relationships)), + relationships=access.relationships, + audit=access.audit, + ) + reads = ArtifactReadSession( + profile.database, "scope", (family, "topic-memory"), access=access, principal=alice, audit=audit + ) + context = GenerationContext( + "scope", resources, reads.read, list_artifacts=reads.list, search_artifacts=reads.search + ) + page = await context.list_artifacts(family, limit=1) + assert [item.artifact_id for item in page.items] == ["b-visible"] + assert page.next_cursor is None + matches = await context.search_artifacts(family, "数据库", limit=1) + assert [hit.artifact_ref.artifact_id for hit in matches.matches] == ["b-visible"] + assert matches.artifacts is not None + assert matches.artifacts[0].content.model_dump() == {"text": "数据库 TIMEOUT"} + assert not reads.refs + snapshot = await context.read_revision(matches.matches[0].artifact_ref) + assert snapshot.content["text"] == "数据库 TIMEOUT" + assert reads.refs == {(family, "b-visible", 1)} + with pytest.raises(AccessDeniedError): + await context.read_latest(family, "a-hidden") + with pytest.raises(InvalidBaseAccessRequestError): + await context.read_latest("undeclared", "any") + with pytest.raises(InvalidBaseAccessRequestError): + await context.list_artifacts("undeclared") + with pytest.raises(InvalidBaseAccessRequestError): + await context.search_artifacts("undeclared", "数据库") + assert await context.read_latest(family, "foreign") is None + denied = ArtifactReadSession( + profile.database, "other", (family,), access=access, principal=alice, audit=audit + ) + foreign_context = GenerationContext( + "other", resources, denied.read, list_artifacts=denied.list, search_artifacts=denied.search + ) + with pytest.raises(AccessDeniedError): + await foreign_context.read_latest(family, "foreign") + with pytest.raises(AccessDeniedError): + await foreign_context.list_artifacts(family) + with pytest.raises(AccessDeniedError): + await foreign_context.search_artifacts(family, "数据库") + topic_page = await context.list_artifacts("topic-memory") + assert topic_page.items == (topic.as_ref(),) + assert (await context.read_revision(topic.as_ref())).content["detail"] == "Evidence" + assert ("topic-memory", "topic", 1) in reads.refs + unauthorized = ArtifactReadSession( + profile.database, "scope", ("topic-memory",), access=access, principal=bob, audit=audit + ) + with pytest.raises(AccessDeniedError): + await unauthorized.list("topic-memory") + with pytest.raises(AccessDeniedError): + await unauthorized.read("topic-memory", "topic", 1) + + asyncio.run(scenario()) + + +def test_generation_context_reads_fixed_history_across_families_and_pages_current_references(tmp_path): + async def scenario(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'history.db'}") + async with AsyncExitStack() as resources: + profile = await resources.enter_async_context(SQLiteProfile.open(config, tables=BUILTIN_TABLES)) + items = custom_artifact_repository("context-items") + roots = custom_artifact_repository("context-roots") + async with profile.database.transaction() as connection: + await SQLiteExperienceFTSIndex().initialize(connection) + for family in ("context-items", "context-roots"): + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={ + "schema_version": "1", + "search_fields": ["text"] if family == "context-items" else [], + }, + ) + ) + first = await items.create( + connection, + "scope", + "a", + RepositoryArtifactDraft( + family="context-items", + content=StoredArtifactContent(schema_version="1", content={"text": "旧证据"}), + ), + ) + await roots.create( + connection, + "scope", + "root", + RepositoryArtifactDraft( + family="context-roots", + content=StoredArtifactContent( + schema_version="1", content={"ref": first.as_ref().model_dump(mode="json")} + ), + ), + ) + for identity in ("b", "c"): + await items.create( + connection, + "scope", + identity, + RepositoryArtifactDraft( + family="context-items", + content=StoredArtifactContent(schema_version="1", content={"text": "其他"}), + ), + ) + builtin = await ArtifactRepository((Profile,)).create( + connection, + "scope", + "profile", + RepositoryArtifactDraft( + family="profile", + content=ProfileContent( + content="使用中文", + generation=ProfileGeneration(mode="manual_create", created_at=datetime.now(UTC)), + ), + ), + ) + await connection.execute( + insert(SOURCE_CURSORS_TABLE).values( + scope_id="scope", binding_name="reports", cursor=b'{"through":7}', generation=3 + ) + ) + reads = ArtifactReadSession(profile.database, "scope", ("context-items", "context-roots", "profile")) + context = GenerationContext( + "scope", resources, reads.read, list_artifacts=reads.list, search_artifacts=reads.search + ) + root = await context.read_latest("context-roots", "root") + assert root is not None + fixed_ref = ArtifactRef.model_validate(root.content["ref"]) + async with profile.database.transaction() as connection: + await items.revise( + connection, + "scope", + first, + RepositoryArtifactDraft( + family="context-items", + content=StoredArtifactContent(schema_version="1", content={"text": "数据库 NEW"}), + ), + ) + builder = ArtifactIndexBuilder(profile.database) + await builder.refresh("scope", "context-items") + assert await builder.build("scope", "context-items") + first_page = await context.list_artifacts("context-items", limit=2) + assert [(ref.artifact_id, ref.revision) for ref in first_page.items] == [("a", 2), ("b", 1)] + assert first_page.next_cursor is not None + last_page = await context.list_artifacts("context-items", limit=2, cursor=first_page.next_cursor) + assert [(ref.artifact_id, ref.revision) for ref in last_page.items] == [("c", 1)] + assert last_page.next_cursor is None + assert not (await context.search_artifacts("context-items", "旧证据")).matches + current = await context.search_artifacts("context-items", "数据库") + assert current.matches[0].artifact_ref.revision == 2 + assert (await context.read_revision(current.matches[0].artifact_ref)).content["text"] == "数据库 NEW" + assert (await context.read_revision(fixed_ref)).content["text"] == "旧证据" + snapshot = await context.read_revision(builtin.as_ref()) + assert snapshot.schema_version is None + assert snapshot.content == builtin.content.model_dump(mode="json", by_alias=True) + with pytest.raises(BaseValueNotFoundError): + await context.read_revision(ArtifactRef(family="context-items", artifact_id="a", revision=99)) + with pytest.raises(ArtifactSearchUnsupported): + await context.search_artifacts("context-items", "数据库", mode="hybrid") + async with profile.database.transaction() as connection: + cursor = (await connection.execute(select(SOURCE_CURSORS_TABLE))).mappings().one() + assert cursor["cursor"] == b'{"through":7}' and cursor["generation"] == 3 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "evidence_family,revoked,update_foreign", + [ + ("topic-memory", False, False), + ("topic-memory", True, False), + ("profile", False, False), + ("profile", True, False), + ("topic-memory", False, True), + ], +) +def test_plugin_commit_rechecks_evidence_permissions(tmp_path, evidence_family, revoked, update_foreign): + async def scenario(): + admin = PrincipalRef(type="service", id="admin") + alice = PrincipalRef(type="user", id="alice") + audit = AccessAuditContext(transport="test", operation="artifact-reads") + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'commit.db'}") + output_family = "context-reports" + register_custom_artifact_access_family(output_family) + async with AsyncExitStack() as resources: + profile = await resources.enter_async_context(SQLiteProfile.open(config, tables=BUILTIN_TABLES)) + access = await resources.enter_async_context( + open_builtin_access_control(config, bootstrap_administrators=(admin,)) + ) + binding = await access.create_binding( + admin, + CreateBinding( + subject=alice, + resource=ResourceRef.scope("scope"), + role=AccessRole.SCOPE_CONTRIBUTOR, + idempotency_key="scope-viewer", + ), + context=audit, + ) + async with profile.database.transaction() as connection: + if evidence_family == "topic-memory": + artifact_type = TopicMemory + content = TopicMemoryContent(title="Topic", summary="Summary", detail="Evidence") + else: + artifact_type = Profile + content = ProfileContent( + content="Evidence", + generation=ProfileGeneration(mode="manual_create", created_at=datetime.now(UTC)), + ) + evidence = await ArtifactRepository((artifact_type,)).create( + connection, + "scope", + "a-hidden", + RepositoryArtifactDraft(family=evidence_family, content=content), + ) + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=output_family, + schema_version="1", + declaration={"schema_version": "1"}, + ) + ) + lease = await ArtifactProcessingLeaseRepository().start_single_process_term(connection, "test-owner") + intent = await ArtifactProcessingIntentRepository().request(connection, "scope", "reports") + assignment = ArtifactProcessingWorkAssignment( + binding_name="reports", + scope_id="scope", + artifact_family=output_family, + claimed_request_generation=intent.requested_generation, + fence=lease.fence("single-process"), + worker_id="test-worker", + ) + security = WorkerSecurity(access, alice) + invocation = ScopeInvocation( + assignment, authorize_transaction=partial(security.authorize_transaction, scope_id="scope") + ) + await invocation.start(connection) + if evidence_family == "profile": + await access.establish_artifact_owner( + ResourceRef.artifact("scope", family=evidence_family, artifact_id="a-hidden"), + alice, + idempotency_key="evidence-owner", + context=audit, + ) + reads = ArtifactReadSession( + profile.database, + "scope", + (evidence_family, output_family), + access=access, + principal=alice, + audit=audit, + guard=invocation.guard, + ) + snapshot = await reads.read(evidence_family, "a-hidden", 1) + assert snapshot.ref == evidence.as_ref() + if revoked: + await access.revoke_binding( + admin, binding.binding_id, expected_version=1, idempotency_key="revoke-scope", context=audit + ) + with pytest.raises(AccessDeniedError): + await reads.read(evidence_family, "a-hidden", 1) + with pytest.raises(AccessDeniedError): + await reads.list(evidence_family) + if update_foreign: + async with profile.database.transaction() as connection: + await custom_artifact_repository(output_family).create( + connection, + "scope", + "report", + RepositoryArtifactDraft( + family=output_family, + content=StoredArtifactContent(schema_version="1", content={"text": "Original"}), + ), + ) + await access.establish_artifact_owner( + ResourceRef.artifact("scope", family=output_family, artifact_id="report"), + PrincipalRef(type="user", id="bob"), + idempotency_key="output-owner", + context=audit, + ) + previous = await reads.read(output_family, "report", 1) + assert previous.content == {"text": "Original"} + draft = ArtifactDraft( + family=output_family, + schema_version="1", + artifact_id="report", + expected_revision=1 if update_foreign else 0, + content={"text": "Result"}, + source_refs=(), + artifact_refs=(snapshot.ref,), + ) + + async def commit(): + await _commit_window( + profile.database, + ArtifactPluginWorkerSpec(config, "reports", 1, {}), + assignment, + {"family": output_family, "schema_version": "1", "content_schema": {"type": "object"}}, + invocation, + SourceRepository(BUILTIN_SOURCE_REGISTRY), + None, + 0, + frozenset(), + reads.refs, + [draft], + security, + ) + + if revoked or update_foreign: + with pytest.raises(AccessDeniedError): + await commit() + else: + await commit() + async with profile.database.transaction() as connection: + repository = custom_artifact_repository(output_family) + if revoked: + with pytest.raises(RepositoryNotFoundError): + await repository.latest(connection, "scope", output_family, "report") + else: + result = await repository.latest(connection, "scope", output_family, "report") + if update_foreign: + assert result.revision == 1 + assert result.content.content == {"text": "Original"} + else: + assert result.lineage.artifacts == (snapshot.ref,) + current = await ArtifactProcessingIntentRepository().load(connection, "scope", "reports") + assert current is not None + assert current.handled_generation == (0 if revoked or update_foreign else intent.requested_generation) + + asyncio.run(scenario()) diff --git a/tests/conftest.py b/tests/conftest.py index be1e0e8ce8..cbc00a5c1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,19 @@ def short_tmp_path() -> Iterator[Path]: yield Path(directory) +@pytest.fixture(autouse=True) +def isolated_artifact_family_profiles(): + """Plugin registration in one test must not change another Server's readiness.""" + from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES + + original = dict(ARTIFACT_FAMILY_PROFILES) + try: + yield + finally: + ARTIFACT_FAMILY_PROFILES.clear() + ARTIFACT_FAMILY_PROFILES.update(original) + + @pytest.fixture(autouse=True) def isolated_client_connection_settings(tmp_path, monkeypatch, request): """Isolate daily runtime settings and persistent setup consent, preserving explicit real E2E runs.""" diff --git a/tests/e2e/test_artifact_authorization_startup.py b/tests/e2e/test_artifact_authorization_startup.py new file mode 100644 index 0000000000..92c58870c5 --- /dev/null +++ b/tests/e2e/test_artifact_authorization_startup.py @@ -0,0 +1,501 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Persisted custom artifacts remain authorized without a plugin manager.""" + +import asyncio + +import httpx +import pytest +from sqlalchemy import insert + +from powercontext.builtin.persistence.artifacts import RepositoryArtifactDraft +from powercontext.builtin.persistence.custom_artifacts import StoredArtifactContent, custom_artifact_repository +from powercontext.builtin.persistence.tables import ARTIFACT_FAMILY_SCHEMAS_TABLE +from powercontext.builtin.runtime.config import RuntimeConfig +from powercontext.server.authentication import StaticBearerAuthenticationProvider +from powercontext.server.authz import AccessAuditContext, AccessRole, CreateBinding, PrincipalRef, ResourceRef +from powercontext.server.authz.composition import open_builtin_access_control +from powercontext.server.authz.profiles import ARTIFACT_FAMILY_PROFILES +from powercontext.server.factory import create_server_app +from powercontext.server.settings import AccessControlConfig, DashboardConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database +from tests.e2e.test_atomic_memory_search_authorization_snapshot import _isolated_database + + +@pytest.mark.parametrize( + "backend,role,mode", + [("sqlite", "all", "global"), ("sqlite", "all", "dedicated"), ("oceanbase", "api", "global")], +) +def test_custom_artifact_authorization_survives_server_restart(tmp_path, monkeypatch, backend, role, mode): + async def scenario(): + family = "startup-report" + admin = PrincipalRef(type="service", id="admin") + reader = PrincipalRef(type="user", id="reader") + async with _isolated_database(backend, tmp_path) as database: + settings = ServerSettings( + database=database, + workspace=tmp_path, + access=AccessControlConfig(mode="enforced"), + runtime=RuntimeConfig( + artifact_processing_families=(), + artifact_processing_role=role, + artifact_processing_supervisor_mode=mode, + ), + dashboard=DashboardConfig(enabled=False), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings, evidence_dir=tmp_path / "evidence") + async with open_builtin_access_control(database, bootstrap_administrators=(admin,)) as access: + + def application(principal): + return create_server_app( + settings=settings, + scheduler_path=tmp_path / "scheduler.db", + access_control=access, + authentication_provider=StaticBearerAuthenticationProvider("test-token", principal), + ) + + app = application(admin) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + scopes = [] + for name in ("visible", "forbidden"): + response = await client.post( + "/v1/scopes", json={"title": name, "summary": name, "idempotency_key": name} + ) + assert response.status_code == 201, response.text + scopes.append(response.json()["scope_id"]) + async with app.state.application._records().database.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={ + "family": family, + "schema_version": "1", + "content_schema": {"type": "object"}, + "search_fields": [], + }, + ) + ) + for scope in scopes: + await custom_artifact_repository(family).create( + connection, + scope, + "report", + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent(schema_version="1", content={"text": "retained"}), + ), + ) + await access.create_binding( + admin, + CreateBinding( + subject=reader, + resource=ResourceRef.scope(scopes[0]), + role=AccessRole.SCOPE_VIEWER, + idempotency_key="reader", + ), + context=AccessAuditContext(transport="test", operation="artifact-startup"), + ) + # Model an empty process-local registry on restart, preserving built-in profiles. + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + discovery_app = application(admin) + async with ( + discovery_app.router.lifespan_context(discovery_app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=discovery_app), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + metadata = await client.get("/v1/capabilities") + assert metadata.status_code == 200, metadata.text + assert family in metadata.json()["artifact_families"] + assert next( + item for item in metadata.json()["custom_artifact_schemas"] if item["family"] == family + ) == { + "family": family, + "schema_version": "1", + "content_schema": {"type": "object"}, + "search_fields": [], + "search": None, + } + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + app = application(reader) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + path = f"/v1/scopes/{scopes[0]}/artifacts/{family}/report" + for suffix in ("", "/revisions/1"): + response = await client.get(path + suffix) + assert response.status_code == 200, response.text + assert response.json()["content"] == {"text": "retained"} + assert response.json()["revision"] == 1 + denied = await client.get(f"/v1/scopes/{scopes[1]}/artifacts/{family}/report") + assert denied.status_code == 403, denied.text + unknown = await client.get(f"/v1/scopes/{scopes[0]}/artifacts/unregistered-family/report") + assert unknown.status_code in {404, 422}, unknown.text + + asyncio.run(scenario()) + + +def test_live_family_discovery_and_access_query_budget(tmp_path, monkeypatch): + """External storage writes do not update this Server's process-local profiles.""" + from sqlalchemy import event + + async def scenario(): + async with _isolated_database("sqlite", tmp_path) as database: + settings = ServerSettings( + database=database, + workspace=tmp_path, + runtime=RuntimeConfig(artifact_processing_families=(), artifact_processing_supervisor_mode="dedicated"), + access=AccessControlConfig(mode="enforced"), + dashboard=DashboardConfig(enabled=False), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=True), + ) + await asyncio.to_thread(prepare_server_database, settings, evidence_dir=tmp_path / "evidence") + admin = PrincipalRef(type="service", id="admin") + async with open_builtin_access_control(database, bootstrap_administrators=(admin,)) as access: + app = create_server_app( + settings=settings, + scheduler_path=tmp_path / "scheduler.db", + access_control=access, + authentication_provider=StaticBearerAuthenticationProvider("test-token", admin), + ) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + response = await client.post( + "/v1/scopes", json={"title": "live", "summary": "live", "idempotency_key": "live"} + ) + assert response.status_code == 201, response.text + scope = response.json()["scope_id"] + db = app.state.application._records().database + queries = [] + binding_reads = [] + assert access.relationships is not None + get_binding = access.relationships.get_binding + + async def counted_get_binding(binding_id): + binding_reads.append(binding_id) + return await get_binding(binding_id) + + monkeypatch.setattr(access.relationships, "get_binding", counted_get_binding) + + def track(_conn, _cursor, statement, _parameters, _context, _many): + if ( + statement.lstrip().upper().startswith("SELECT") + and "pc_artifact_family_schemas" in statement + ): + queries.append(statement) + + event.listen(db.engine.sync_engine, "before_cursor_execute", track) + try: + for entry in ( + "read", + "capabilities", + "access/check", + "roles/list", + "bindings/list", + "bindings/create", + "resources/list", + "bindings/revoke", + "bindings/replace", + ): + family = "live-" + entry.replace("/", "-") + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + # Model a different process committing storage without touching this registry. + async with db.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={ + "family": family, + "schema_version": "1", + "content_schema": {}, + "search_fields": [], + }, + ) + ) + await custom_artifact_repository(family).create( + connection, + scope, + "report", + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent(schema_version="1", content={"text": "live"}), + ), + ) + assert family not in ARTIFACT_FAMILY_PROFILES + path = f"/v1/scopes/{scope}/artifacts/{family}/report" + queries.clear() + await _assert_family_display(client, family, present=False) + assert queries == [], "Family display must not refresh the schema registry" + binding_reads.clear() + result = await _request_family_operation(client, access, admin, entry, family, scope, path) + expected_status = ( + 422 if entry in {"bindings/create", "bindings/replace", "resources/list"} else 200 + ) + assert result.status_code == expected_status, result.text + assert len(queries) == (2 if entry == "read" else 1), queries + _assert_discovered_family(entry, family, result.json()) + if entry in {"bindings/revoke", "bindings/replace"}: + assert binding_reads == [family], "Binding ID must be resolved only once" + if entry in { + "roles/list", + "bindings/list", + "bindings/create", + "resources/list", + "bindings/replace", + }: + queries.clear() + repeated = await _request_family_operation( + client, access, admin, entry, family, scope, path + ) + assert repeated.status_code == result.status_code, repeated.text + assert repeated.json() == result.json() + assert queries == [], "Known family management must not refresh the registry" + queries.clear() + await _assert_family_display(client, family, present=True) + assert queries == [], "Family display must use the local registry" + for suffix in ("", "/revisions/1"): + read = await client.get(path + suffix) + assert read.status_code == 200, read.text + assert len(queries) == 2, queries # Existing persisted-family validation: one per read. + finally: + event.remove(db.engine.sync_engine, "before_cursor_execute", track) + + asyncio.run(scenario()) + + +def _assert_discovered_family(entry, family, payload): + if entry == "capabilities": + assert family in payload["artifact_families"] + assert any(item["family"] == family for item in payload["custom_artifact_schemas"]) + elif entry == "access/check": + assert payload["allowed"] is True + + +async def _assert_family_display(client, family, *, present): + response = await client.get("/v1/access/me") + assert response.status_code == 200, response.text + assert any(item["family"] == family for item in response.json()["artifact_families"]) is present + response = await client.get("/health/ready") + assert response.status_code == 200, response.text + assert (f"{family}:enabled" in response.json()["checks"]["access_artifact_families"]) is present + + +async def _request_family_operation(client, access, admin, entry, family, scope, path): + if entry == "read": + return await client.get(path) + if entry == "capabilities": + return await client.get("/v1/capabilities") + resource = {"type": "artifact", "scope_id": scope, "identity": {"family": family, "artifact_id": "report"}} + subject = {"type": "user", "id": "recipient"} + payloads = { + "access/check": {"requirements": [{"action": "artifact.read", "resource": resource}], "match": "all"}, + "roles/list": {"resource_type": "artifact", "family": family}, + "bindings/list": {"management_resource": resource}, + "bindings/create": { + "resource": resource, + "subject": subject, + "role": "artifact.viewer", + "idempotency_key": family, + }, + "resources/list": {"resource_type": "artifact", "family": family, "action": "artifact.read"}, + } + if entry in {"bindings/revoke", "bindings/replace"}: + await _seed_retained_binding(access, admin, family, scope) + payload = {"binding_id": family, "expected_version": 1, "idempotency_key": family + "-change"} + if entry == "bindings/replace": + payload["replacement"] = {"subject": subject} + else: + payload = payloads[entry] + endpoint = entry if entry == "access/check" else "access/" + entry + return await client.post("/v1/" + endpoint, json=payload) + + +async def _seed_retained_binding(access, admin, family, scope): + from datetime import UTC, datetime + + from powercontext.server.authz.models import AccessBinding, AccessBindingState + + # Exercise an existing relationship row; custom families still cannot grant direct roles through HTTP. + await access.relationships.create_binding( + AccessBinding( + binding_id=family, + subject=admin, + resource=ResourceRef.artifact(scope, family=family, artifact_id="report"), + role=AccessRole.ARTIFACT_OWNER, + granted_by=admin, + reason=None, + created_at=datetime.now(UTC), + expires_at=None, + state=AccessBindingState.ACTIVE, + version=1, + policy_revision="pending", + idempotency_key=family, + ) + ) + + +class _FilteringProvider: + """A custom Provider that retains Scope grants but denies one exact Artifact.""" + + def __init__(self, delegate): + self.delegate = delegate + + async def check(self, request): + from dataclasses import replace + + decision = await self.delegate.check(request) + if request.resource.identity is not None and request.resource.identity.artifact_id == "hidden": + return replace( + decision, allowed=False, reason_code="no-matching-policy", matched_subject=None, matched_binding_id=None + ) + return decision + + async def check_batch(self, requests): + return tuple([await self.check(request) for request in requests]) + + async def resolve_resource_filter(self, request): + return await self.delegate.resolve_resource_filter(request) + + +def test_first_search_loads_external_family_with_custom_provider(tmp_path, monkeypatch): + from sqlalchemy import event + + from powercontext.artifacts.indexes import ArtifactSearchDeclaration, TextIndex + from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder + + async def scenario(): + family = "live-search-report" + monkeypatch.delitem(ARTIFACT_FAMILY_PROFILES, family, raising=False) + async with _isolated_database("sqlite", tmp_path) as database: + settings = ServerSettings( + database=database, + workspace=tmp_path, + access=AccessControlConfig(mode="enforced"), + runtime=RuntimeConfig(artifact_processing_families=(), artifact_processing_supervisor_mode="dedicated"), + dashboard=DashboardConfig(enabled=False), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings, evidence_dir=tmp_path / "evidence") + admin = PrincipalRef(type="service", id="admin") + async with open_builtin_access_control(database, bootstrap_administrators=(admin,)) as access: + access.provider = _FilteringProvider(access.provider) + app = create_server_app( + settings=settings, + scheduler_path=tmp_path / "scheduler.db", + access_control=access, + authentication_provider=StaticBearerAuthenticationProvider("test-token", admin), + ) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://testserver", + headers={"Authorization": "Bearer test-token"}, + ) as client, + ): + response = await client.post( + "/v1/scopes", json={"title": "search", "summary": "search", "idempotency_key": "search"} + ) + assert response.status_code == 201, response.text + scope = response.json()["scope_id"] + db = app.state.application._records().database + search = ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="text", fields=("text",), projection_version="1"),), + modes=("text",), + default_mode="text", + ) + # Commit external storage changes without touching this Server's profile registry. + async with db.transaction() as connection: + await connection.execute( + insert(ARTIFACT_FAMILY_SCHEMAS_TABLE).values( + family=family, + schema_version="1", + declaration={ + "family": family, + "schema_version": "1", + "content_schema": {}, + "search_fields": [], + "search": search.model_dump(mode="json"), + }, + ) + ) + for identity in ("visible", "hidden"): + await custom_artifact_repository(family).create( + connection, + scope, + identity, + RepositoryArtifactDraft( + family=family, + content=StoredArtifactContent( + schema_version="1", content={"text": "database timeout"} + ), + ), + ) + builder = ArtifactIndexBuilder(db) + await builder.refresh(scope, family) + assert await builder.build(scope, family) + assert family not in ARTIFACT_FAMILY_PROFILES + queries = [] + + def track(_conn, _cursor, statement, _parameters, _context, _many): + if ( + statement.lstrip().upper().startswith("SELECT") + and "pc_artifact_family_schemas" in statement + ): + queries.append(statement) + + event.listen(db.engine.sync_engine, "before_cursor_execute", track) + try: + path = f"/v1/scopes/{scope}/artifacts/{family}/search" + first = await client.post(path, json={"query": "timeout"}) + assert first.status_code == 200, first.text + assert [item["artifact_id"] for item in first.json()["results"]] == ["visible"] + first_queries = len(queries) + queries.clear() + second = await client.post(path, json={"query": "timeout"}) + assert second.status_code == 200, second.text + assert second.json() == first.json() + assert len(queries) == first_queries - 1, "Only the first search may load the missing profile" + finally: + event.remove(db.engine.sync_engine, "before_cursor_execute", track) + + asyncio.run(scenario()) diff --git a/tests/e2e/test_artifact_index_backends.py b/tests/e2e/test_artifact_index_backends.py new file mode 100644 index 0000000000..cf95d858d4 --- /dev/null +++ b/tests/e2e/test_artifact_index_backends.py @@ -0,0 +1,105 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real backend projection transactions, native initialization and exact search.""" + +import asyncio +import json + +import pytest + +from powercontext.artifacts.search import ArtifactSearchUnavailable +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder, ArtifactProjectionPublisher +from powercontext.builtin.persistence.oceanbase import OceanBaseProfile +from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.builtin.runtime.custom_artifact_search import CustomArtifactSearcher, CustomArtifactSearchRequest +from tests.builtin.persistence.test_artifact_indexes import PROFILE, body, declaration, register, service, write + + +def test_search_projection_backends_require_native_provisioning_and_preserve_revisions(migration_backend): + config = migration_backend.config + + def open_storage(): + if isinstance(config, SQLiteConfig): + return SQLiteProfile.open(config, tables=BUILTIN_TABLES, load_vector_extension=True) + if isinstance(config, SeekDBConfig): + return SeekDBProfile.open(config, tables=BUILTIN_TABLES) + return OceanBaseProfile.open(config, tables=BUILTIN_TABLES) + + async def declare(): + async with open_storage() as storage: + db = storage.database + await register(db, "reports", declaration()) + publisher = ArtifactProjectionPublisher(db) + await write(db, publisher, "scope", "reports", "a", body("数据库 timeout", *([[0, 0]] * 8))) + await write(db, publisher, "scope", "reports", "b", body("数据库 normal", [1, 0])) + builder = ArtifactIndexBuilder(db) + await builder.refresh("scope", "reports") + if isinstance(config, SQLiteConfig): + assert await builder.build("scope", "reports") + else: + with pytest.raises(ArtifactSearchUnavailable, match="database initialization"): + await builder.build("scope", "reports") + + asyncio.run(declare()) + + async def verify(): + async with open_storage() as storage: + db = storage.database + builder = ArtifactIndexBuilder(db) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + search = service(db, "reports") + vector = {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]} + request = {"mode": "vector", "query_vector": vector, "limit": 2, "include_scores": True} + found = await search.search("scope", "reports", request) + assert [item.artifact_id for item in found.artifacts] == ["a", "b"] + assert found.matches[0].channel_scores["vector"].raw == 0 + hybrid = await search.search("scope", "reports", {**request, "mode": "hybrid", "query": "数据库"}) + assert {item.artifact_id for item in hybrid.artifacts} == {"a", "b"} + # Pause only to order the concurrent operations; all reads and writes use the real backend. + reading, published = asyncio.Event(), asyncio.Event() + + class SnapshotReader(CustomArtifactSearcher): + async def _candidates(self, *args, **kwargs): + reading.set() + await published.wait() + return await super()._candidates(*args, **kwargs) + + reader = asyncio.create_task( + SnapshotReader(db, "reports").search( + "scope", + CustomArtifactSearchRequest.model_validate_json( + json.dumps({**request, "mode": "hybrid", "query": "数据库"}) + ), + ) + ) + try: + await asyncio.wait_for(reading.wait(), 10) + await builder.refresh("scope", "reports", rebuild=True) + with pytest.raises(ArtifactSearchUnavailable): + await search.search("scope", "reports", request) + assert await builder.build("scope", "reports") + finally: + published.set() + snapshot = await reader + assert {item.artifact_id for item in snapshot.artifacts} == {"a", "b"} + publisher = ArtifactProjectionPublisher(db) + await write(db, publisher, "scope", "reports", "a", body("recovered")) + assert [item.artifact_id for item in (await search.search("scope", "reports", request)).artifacts] == ["b"] + assert not (await search.search("scope", "reports", {"query": "timeout"})).matches + + asyncio.run(verify()) diff --git a/tests/e2e/test_artifact_plugin_search.py b/tests/e2e/test_artifact_plugin_search.py new file mode 100644 index 0000000000..25ad575733 --- /dev/null +++ b/tests/e2e/test_artifact_plugin_search.py @@ -0,0 +1,412 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Custom Artifact search through generation, revisions, unload and index recovery.""" + +import asyncio +import json +import shutil +from pathlib import Path + +from powercontext.builtin.persistence.artifact_indexes import ArtifactIndexBuilder +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from tests.e2e.test_artifact_plugins import ( + EXAMPLE, + REGISTRATION, + _wait_for_search_ready, + change, + create_scope, + deploy_plugin, + enable_all, + plugin_state, + publish_plugin, + reload_plugin, + running_server, +) + +_PIPELINE = """ + +class SearchPipeline: + def __init__(self, ctx): + self.ctx = ctx + + async def generate(self, batch): + drafts = [] + for source in batch.sources: + value = json.loads(TextEvidence.model_validate(source.value).content) + artifact_id = value.pop("artifact_id") + previous = await self.ctx.read_latest("incident-report", artifact_id) + drafts.append(ArtifactDraft( + family="incident-report", schema_version=families[0].schema_version, + artifact_id=artifact_id, + expected_revision=0 if previous is None else previous.ref.revision, + content=value, source_refs=(source.ref,), + )) + return drafts + +async def create_pipeline(ctx, config): + return SearchPipeline(ctx) +""" + + +async def _write_report(client, scope, source_id, artifact_id, body, *, title="Report", revision=1, fragments=None): + response = await client.post( + "/v1/sources/content", + json={ + "scope_id": scope, + "source_id": source_id, + "content": json.dumps( + { + "artifact_id": artifact_id, + "title": title, + "evidence_text": body, + "private_text": "不可检索 secretword", + **({"fragments": fragments} if fragments is not None else {}), + }, + ensure_ascii=False, + ), + }, + ) + assert response.status_code == 202, response.text + async with asyncio.timeout(45): + while True: + page = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report") + assert page.status_code == 200, page.text + if any( + item["artifact_id"] == artifact_id and item["revision"] == revision for item in page.json()["items"] + ): + response = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{artifact_id}") + assert response.status_code == 200, response.text + return response.json() + await asyncio.sleep(0.2) + + +async def _search(client, scope, query): + response = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": query}) + assert response.status_code == 200, response.text + return response.json() + + +def test_custom_search_preserves_full_content_and_rebuilds_retained_revisions(tmp_path: Path) -> None: + asyncio.run(_search_through_lifecycle(tmp_path)) + + +async def _search_through_lifecycle(directory: Path) -> None: + async with running_server(directory) as client: + scope = await create_scope(client, "search") + other_scope = await create_scope(client, "other-search") + plugin = deploy_plugin(EXAMPLE, directory / "search-v1", [scope, other_scope]) + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text().replace( + "class IncidentReport(BaseModel):", "class IncidentReport(BaseModel):\n private_text: str" + ) + + _PIPELINE, + encoding="utf-8", + ) + await reload_plugin(client, directory, plugin, 0) + await enable_all(client) + state = await plugin_state(client, "enabled") + removed = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/query", json={"query": "数据库"}) + assert removed.status_code == 405 # The remaining GET /{artifact_id} route has no POST operation. + for invalid in ( + {"query": "数据库", "limit": 201}, + {"query": "数据库", "indexes": {"text": []}}, + {"query": "数据库", "mode": "fts"}, + ): + response = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/search", json=invalid) + assert response.status_code == 422, response.text + + original = "数据库连接超时 TIMEOUT，重试后恢复。" # noqa: RUF001 + first = await _write_report(client, scope, "first", "report", original, title="故障记录") + # The first write creates the durable build request; empty scopes no longer + # receive prebuilt releases when the plugin is loaded. + await _wait_for_search_ready(client, scope, "数据库") + for query in ("数据库", "连接超时", "timeout", "数据库 timeout", "故障记录"): + hits = (await _search(client, scope, query))["results"] + assert [{key: hit[key] for key in ("family", "artifact_id", "revision")} for hit in hits] == [ + {"family": "incident-report", "artifact_id": "report", "revision": 1} + ] + assert hits[0]["content"] == first["content"] + assert set(hits[0]) == {"family", "artifact_id", "revision", "content", "lineage"} + for query in ("不存在的英文token", "secretword", "不可检索", "!!!"): + assert (await _search(client, scope, query))["results"] == [] + assert (await _search(client, other_scope, "数据库"))["results"] == [] + + # Normalization can change code point counts; excerpts must still use the exact original. + for index, (needle, query) in enumerate(( + ("磁盘告警", "磁盘告警"), + ("Straße", "strasse"), + ("Cafe\u0301", "café"), + ("customer_id", "customer_id"), + ("pcu123", "pcu123"), + )): + body = "prefix ß e\u0301. " * 80 + needle + " suffix." * 80 + await _write_report(client, scope, f"long-{index}", f"long-{index}", body) + hits = (await _search(client, scope, query))["results"] + assert len(hits) == 1 + assert hits[0]["content"]["evidence_text"] == body + for query in ("cafe", "customer", "id"): + assert (await _search(client, scope, query))["results"] == [] + + # Replacing a head must remove its old words while retaining the exact historical revision. + await _write_report(client, scope, "updated", "report", "网络恢复正常 recovered", revision=2) + assert (await _search(client, scope, "数据库 timeout"))["results"] == [] + assert (await _search(client, scope, "网络恢复"))["results"][0]["revision"] == 2 + historical = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/report/revisions/1") + assert historical.status_code == 200 + assert historical.json()["content"] == first["content"] + + # A schema bump must not bypass the fixed search declaration. + replacement = deploy_plugin(plugin, directory / "search-v2", [scope, other_scope], version="2") + entry = replacement / "plugin.py" + entry.write_text( + entry + .read_text() + .replace('schema_version="1"', 'schema_version="2"') + .replace('fields=("title", "evidence_text")', 'fields=("evidence_text",)'), + encoding="utf-8", + ) + publish_plugin(directory, replacement) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await _search(client, scope, "网络恢复"))["results"][0]["revision"] == 2 + # Code-only reload remains supported with exactly the same search declaration. + entry.write_text((plugin / "plugin.py").read_text()) + state = await reload_plugin(client, directory, replacement, state["binding_generation"], "enabled") + await _write_report(client, scope, "after-reload", "after-reload", "缓存刷新 complete") + await _wait_for_search_ready(client, scope, "缓存刷新") + disabled = await change(client, "disable", state["binding_generation"], "disabled") + assert (await _search(client, scope, "缓存刷新"))["results"][0]["content"][ + "evidence_text" + ] == "缓存刷新 complete" + await change(client, "unload", disabled["binding_generation"], "unloaded") + before = { + query: await _search(client, scope, query) + for query in ("网络恢复", "磁盘告警", "strasse", "café", "customer_id", "pcu123", "缓存刷新") + } + saved = (await client.get(f"/v1/scopes/{scope}/artifacts/incident-report")).json() + + # Persist a rebuild request, then recover it without either plugin's code. + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{directory / 'server.db'}"), tables=BUILTIN_TABLES + ) as storage: + await ArtifactIndexBuilder(storage.database).refresh(scope, "incident-report", rebuild=True) + shutil.rmtree(plugin) + shutil.rmtree(replacement) + async with running_server(directory) as client: + assert (await client.get(REGISTRATION)).json()["phase"] == "unloaded" + assert (await client.get(f"/v1/scopes/{scope}/artifacts/incident-report")).json() == saved + # Server startup no longer waits for rebuilding retained data. Only the + # affected search can be unavailable until its complete manifest is published. + await _wait_for_search_ready(client, scope, "网络恢复") + for query, expected in before.items(): + assert await _search(client, scope, query) == expected + assert (await _search(client, scope, "数据库 timeout hiddenfield secretword"))["results"] == [] + assert (await client.get(f"/v1/scopes/{scope}/artifacts/incident-report")).json() == saved + + +def test_pending_vector_index_suspends_all_search_but_not_generation(tmp_path): + async def run(): + async with running_server(tmp_path) as client: + scope = await create_scope(client, "pending-search") + plugin = deploy_plugin(EXAMPLE, tmp_path / "initial", [scope]) + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text().replace( + "class IncidentReport(BaseModel):", "class IncidentReport(BaseModel):\n private_text: str" + ) + + _PIPELINE + ) + entry.write_text( + entry.read_text() + + """ +from dataclasses import replace +from powercontext.artifact_plugins import VectorIndex +from powercontext.builtin.artifacts.memory import EmbeddingProfile +families = (replace(families[0], search=ArtifactSearchDeclaration( + version="2", indexes=(*families[0].search.indexes, VectorIndex( + name="semantic", projection_version="1", source="text", fields=("evidence_text",), + profile=EmbeddingProfile(profile_id="unavailable", model="unconfigured", dimension=2), + )), modes=("text", "vector"), default_mode="text", +)),) +""" + ) + await reload_plugin(client, tmp_path, plugin, 0) + await enable_all(client) + await plugin_state(client, "enabled") + await _write_report(client, scope, "first", "report", "outage") + response = await client.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"mode": "vector", "query": "outage"} + ) + assert response.status_code == 503 + response = await client.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": "outage"} + ) + assert response.status_code == 503 + latest = await _write_report(client, scope, "second", "report", "recovered", revision=2) + assert latest["revision"] == 2 + historical = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/report/revisions/1") + assert historical.status_code == 200, historical.text + assert historical.json()["revision"] == 1 + assert historical.json()["content"]["evidence_text"] == "outage" + response = await client.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": "recovered"} + ) + assert response.status_code == 503 + + asyncio.run(run()) + + +def test_index_definitions_stay_fixed_while_query_policy_survives_reload_and_restart(tmp_path): + from tests.e2e.test_artifact_plugins import artifacts, capture + + def deploy(version, profile_id, index_name="visual"): + plugin = deploy_plugin(EXAMPLE, tmp_path / version, None, version=version) + entry = plugin / "plugin.py" + entry.write_text( + entry + .read_text() + .replace( + "class IncidentReport(BaseModel):", + "from powercontext.builtin.artifacts.memory import EmbeddingProfile\n" + f'VECTOR_PROFILE = EmbeddingProfile(profile_id="{profile_id}", model="fixture", dimension=2)\n' + "class IncidentReport(BaseModel):\n vector: list[float]\n profile: EmbeddingProfile", + ) + .replace('title=f"', 'vector=[1.0, 0.0], profile=VECTOR_PROFILE, title=f"') + + f''' +from dataclasses import replace +from powercontext.artifact_plugins import VectorIndex +families = (replace(families[0], search=ArtifactSearchDeclaration( + version="1", indexes=(*families[0].search.indexes, VectorIndex( + name="{index_name}", projection_version="1", source="content", + vector_field="vector", profile_field="profile", profile=VECTOR_PROFILE, + )), modes=("text", "vector", "hybrid"), default_mode="text", +)),) +''' + ) + return plugin + + async def run(): + async with running_server(tmp_path) as client: + scope = await create_scope(client, "profile-admission") + first = deploy("1", "features-v1") + state = await reload_plugin(client, tmp_path, first, 0) + # No content exists yet; the declaration is nevertheless already fixed. + second = deploy("2", "features-v2") + publish_plugin(tmp_path, second) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await client.get(REGISTRATION)).json() == state + publish_plugin(tmp_path, first) + await enable_all(client) + state = await plugin_state(client, "enabled") + await capture(client, scope, "original") + items = await artifacts(client, scope, 1) + await _wait_for_search_ready(client, scope, "Outage") + state = await change(client, "disable", state["binding_generation"], "disabled") + before = await _search(client, scope, "Outage") + vector_request = { + "mode": "vector", + "query_vector": { + "profile": before["results"][0]["content"]["profile"], + "vector": [1.0, 0.0], + }, + } + url = f"/v1/scopes/{scope}/artifacts/incident-report/search" + vector_before = await client.post(url, json=vector_request) + assert vector_before.status_code == 200, vector_before.text + assert vector_before.json()["results"] + # Neither profile, name, projection version, schema identity nor index membership may change. + edits = ( + ("features-v1", "features-v2"), + ('name="visual"', 'name="renamed"'), + ('projection_version="1"', 'projection_version="2"'), + ('schema_version="1"', 'schema_version="2"'), + ('version="1", indexes=', 'version="2", indexes='), + ("indexes=(*families[0].search.indexes, VectorIndex(", "indexes=(VectorIndex("), + ) + for number, (old, new) in enumerate(edits, start=3): + replacement = deploy(str(number), "features-v1") + entry = replacement / "plugin.py" + assert old in entry.read_text() + code = entry.read_text().replace(old, new) + if old.startswith("indexes="): + code = code.replace(")), modes=", "),), modes=") + code = code.replace( + 'modes=("text", "vector", "hybrid"), default_mode="text"', + 'modes=("vector",), default_mode="vector"', + ) + entry.write_text(code) + publish_plugin(tmp_path, replacement) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await client.get(REGISTRATION)).json() == state + assert await _search(client, scope, "Outage") == before + unchanged = await client.post(url, json=vector_request) + assert unchanged.status_code == 200, unchanged.text + assert unchanged.json() == vector_before.json() + assert await artifacts(client, scope, 1) == items + policy = deploy("query-policy", "features-v1") + entry = policy / "plugin.py" + entry.write_text( + entry.read_text().replace( + 'modes=("text", "vector", "hybrid"), default_mode="text",', + 'modes=("text", "vector"), default_mode="vector", ' + 'defaults={"text": {"text": ("lexical",)}, "vector": {"vector": ("visual",)}}, ' + "candidate_limit=17, rrf_k=11, text_weight=2, vector_weight=3,", + ) + ) + state = await reload_plugin(client, tmp_path, policy, state["binding_generation"]) + default_vector = {"query_vector": vector_request["query_vector"]} + result = await client.post(url, json=default_vector) + assert result.status_code == 200, result.text + assert result.json() == vector_before.json() + assert (await client.post(url, json={"mode": "text", "query": "Outage"})).json() == before + assert (await client.post(url, json={"mode": "hybrid", "query": "Outage"})).status_code == 422 + assert await artifacts(client, scope, 1) == items + # Invalid policy must not replace the last accepted configuration. + entry.write_text(entry.read_text().replace('("visual",)}', '("missing",)}')) + publish_plugin(tmp_path, policy) + invalid = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert invalid.status_code == 422, invalid.text + assert (await client.get(REGISTRATION)).json() == state + assert (await client.post(url, json=default_vector)).json() == vector_before.json() + state = await change(client, "unload", state["binding_generation"], "unloaded") + async with running_server(tmp_path) as client: + # Unload clears the runtime definition, but the persisted contract still rejects changes. + publish_plugin(tmp_path, second) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 409, rejected.text + assert (await client.get(REGISTRATION)).json() == state + assert (await client.post(url, json={"mode": "text", "query": "Outage"})).json() == before + assert (await client.post(url, json=default_vector)).json() == vector_before.json() + assert (await client.post(url, json={"mode": "hybrid", "query": "Outage"})).status_code == 422 + compatible = deploy("code-only", "features-v1") + state = await reload_plugin(client, tmp_path, compatible, state["binding_generation"]) + assert await _search(client, scope, "Outage") == before + assert (await client.get(REGISTRATION)).json()["binding_generation"] == state["binding_generation"] + + asyncio.run(run()) diff --git a/tests/e2e/test_artifact_plugins.py b/tests/e2e/test_artifact_plugins.py new file mode 100644 index 0000000000..85df6a2f28 --- /dev/null +++ b/tests/e2e/test_artifact_plugins.py @@ -0,0 +1,1023 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Use a delivered plugin through the installed Server CLI and public HTTP APIs.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import socket +import sys +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import httpx +import yaml + +EXAMPLE = Path(__file__).resolve().parents[2] / "examples/artifact_plugins/incident_report" +SUMMARY_EXAMPLE = EXAMPLE.parent / "ticket_summary" +ADMIN_AUTHORIZATION_VALUE = "artifact-plugin-e2e-admin" +REGISTRATION = "/v1/admin/artifact-plugins/incident-reports" +SUMMARY_REGISTRATION = "/v1/admin/artifact-plugins/ticket-summaries" + + +def deploy_plugin(source: Path, directory: Path, scope_ids: list[str] | None, **parameters: Any) -> Path: + """Prepare a deployed version outside the discovery directory.""" + + shutil.copytree(source, directory, ignore=shutil.ignore_patterns("__pycache__")) + path = directory / "plugin.yml" + manifest = yaml.safe_load(path.read_text()) + manifest.update(parameters) + if scope_ids is None: + manifest.pop("scope_ids", None) + else: + manifest["scope_ids"] = scope_ids + path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8") + return directory + + +def publish_plugin(directory: Path, plugin: Path, binding_name: str | None = None) -> Path: + """Switch a user's discovery entry without changing any previous version.""" + + if binding_name is None: + binding_name = yaml.safe_load((plugin / "plugin.yml").read_text())["binding_name"] + root = directory / "plugins" + root.mkdir(exist_ok=True) + entry = root / binding_name + entry.unlink(missing_ok=True) + entry.symlink_to(plugin, target_is_directory=True) + return entry + + +async def reload_plugin( + client: httpx.AsyncClient, + directory: Path, + plugin: Path, + generation: int, + phase: str = "disabled", + registration: str = REGISTRATION, +) -> dict[str, Any]: + publish_plugin(directory, plugin, registration.rsplit("/", 1)[1]) + response = await client.post(f"{registration}/reload", json={"expected_generation": generation}) + assert response.status_code == 202, response.text + assert response.headers["location"] == registration + return await plugin_state(client, phase, registration) + + +async def enable_all( + client: httpx.AsyncClient, *, expected_failures: frozenset[str] = frozenset() +) -> dict[str, dict[str, Any]]: + response = await client.post("/v1/admin/artifact-plugins/enable-all") + assert response.status_code == 200, response.text + items = response.json()["items"] + failed = frozenset(item["binding_name"] for item in items if item["outcome"] == "failed") + assert failed == expected_failures, response.text + assert all(item["status_url"] == f"/v1/admin/artifact-plugins/{item['binding_name']}" for item in items) + return {item["binding_name"]: item for item in items} + + +@asynccontextmanager +async def running_server( + directory: Path, + *, + envfile: Path | None = None, + directory_from_env: bool = True, + database_url: str | None = None, + database_settings: dict[str, Any] | None = None, +): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + env = {key: value for key, value in os.environ.items() if not key.startswith("POWERCONTEXT_SERVER_")} + env.update({ + "POWERCONTEXT_SERVER_DATABASE": json.dumps( + database_settings + or { + "kind": "sqlite", + "url": database_url or f"sqlite+aiosqlite:///{directory / 'server.db'}", + } + ), + "POWERCONTEXT_SERVER_RUNTIME": json.dumps({"artifact_processing_families": []}), + "POWERCONTEXT_SERVER_WORKSPACE": str(directory), + "POWERCONTEXT_HOME": str(directory / "home"), + "POWERCONTEXT_SERVER_AUTH_ENABLED": "true", + "POWERCONTEXT_SERVER_AUTH_TOKEN": ADMIN_AUTHORIZATION_VALUE, + "POWERCONTEXT_SERVER_ACCESS_MODE": "enforced", + "POWERCONTEXT_SERVER_MCP_ENABLED": "false", + "POWERCONTEXT_SERVER_METRICS_ENABLED": "true", + "POWERCONTEXT_SERVER_DASHBOARD_ENABLED": "false", + }) + (directory / "plugins").mkdir(exist_ok=True) + if directory_from_env: + env["POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR"] = str(directory / "plugins") + envfile_args = ["--no-env-file"] if envfile is None else ["--env-file", str(envfile)] + entrypoint = Path(sys.executable).with_name("powercontext") + with (directory / "server.log").open("a") as log: + process = await asyncio.create_subprocess_exec( + sys.executable, + str(entrypoint), + "server", + "run", + *envfile_args, + "--host", + "127.0.0.1", + "--port", + str(port), + cwd=directory, + env=env, + stdout=log, + stderr=log, + ) + try: + async with httpx.AsyncClient( + base_url=f"http://127.0.0.1:{port}", + headers={"Authorization": f"Bearer {ADMIN_AUTHORIZATION_VALUE}"}, + timeout=20, + ) as client: + async with asyncio.timeout(40): + while True: + assert process.returncode is None, (directory / "server.log").read_text() + try: + response = await client.get("/health/ready") + if response.status_code == 200: + break + except httpx.ConnectError: + pass + await asyncio.sleep(0.1) + yield client + finally: + if process.returncode is None: + process.terminate() + try: + await asyncio.wait_for(process.wait(), 20) + except TimeoutError: + process.kill() + await process.wait() + + +async def create_scope(client: httpx.AsyncClient, key: str) -> str: + response = await client.post( + "/v1/scopes", json={"title": key, "summary": "Artifact plugin acceptance", "idempotency_key": key} + ) + assert response.status_code == 201, response.text + return response.json()["scope_id"] + + +async def capture(client: httpx.AsyncClient, scope: str, source_id: str) -> None: + response = await client.post( + "/v1/sources/content", + json={ + "scope_id": scope, + "source_id": source_id, + "content": f"Outage {source_id}: database recovered after retry.", + }, + ) + assert response.status_code == 202, response.text + + +async def plugin_state(client: httpx.AsyncClient, phase: str, registration: str = REGISTRATION) -> dict[str, Any]: + async with asyncio.timeout(40): + while True: + response = await client.get(registration) + assert response.status_code == 200, response.text + state = response.json() + assert state["phase"] != "failed" or phase == "failed", state + if state["phase"] == phase: + return state + await asyncio.sleep(0.1) + + +async def artifacts( + client: httpx.AsyncClient, scope: str, count: int, family: str = "incident-report" +) -> list[dict[str, Any]]: + async with asyncio.timeout(45): + while True: + response = await client.get(f"/v1/scopes/{scope}/artifacts/{family}") + assert response.status_code == 200, response.text + items = response.json()["items"] + if len(items) == count: + return items + await asyncio.sleep(0.2) + + +async def query(client: httpx.AsyncClient, scope: str, family: str = "incident-report") -> dict[str, Any]: + response = await client.post(f"/v1/scopes/{scope}/artifacts/{family}/search", json={"query": "Outage"}) + assert response.status_code == 200, response.text + return response.json() + + +async def _wait_for_search_ready(client, scope, query): + async with asyncio.timeout(30): + while True: + response = await client.post(f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": query}) + if response.status_code != 503: + assert response.status_code == 200, response.text + return + assert response.json()["error"]["code"] == "artifact_search_unavailable", response.text + await asyncio.sleep(0.1) + + +async def change( + client: httpx.AsyncClient, action: str, generation: int, phase: str, registration: str = REGISTRATION +) -> dict[str, Any]: + response = await client.post(f"{registration}/{action}", json={"expected_generation": generation}) + assert response.status_code == 202, response.text + assert response.headers["location"] == registration + return await plugin_state(client, phase, registration) + + +async def wait_for_worker_metric(client: httpx.AsyncClient, metric: str, family: str = "incident-report") -> None: + async with asyncio.timeout(40): + while True: + response = await client.get("/metrics") + assert response.status_code == 200, response.text + if any( + f'family="{family}"' in line and float(line.rsplit(" ", 1)[1]) > 0 + for line in response.text.splitlines() + if line.startswith(f"powercontext_server_artifact_processing_{metric}{{") + ): + return + await asyncio.sleep(0.2) + + +async def worker_metric(client: httpx.AsyncClient, metric: str, family: str) -> float: + response = await client.get("/metrics") + assert response.status_code == 200, response.text + values = [ + float(line.rsplit(" ", 1)[1]) + for line in response.text.splitlines() + if line.startswith(f'powercontext_server_artifact_processing_{metric}{{family="{family}"}} ') + ] + assert len(values) == 1, response.text + return values[0] + + +async def contents(client: httpx.AsyncClient, scope: str, family: str, count: int) -> list[dict[str, Any]]: + result = [] + for item in await artifacts(client, scope, count, family): + response = await client.get(f"/v1/scopes/{scope}/artifacts/{family}/{item['artifact_id']}") + assert response.status_code == 200, response.text + result.append(response.json()) + return result + + +def test_strict_plugin_configuration_survives_generation_reload_and_restart(tmp_path: Path) -> None: + asyncio.run(_use_strict_plugin_configuration(tmp_path)) + + +async def _use_strict_plugin_configuration(tmp_path: Path) -> None: + plugin = deploy_plugin(EXAMPLE, tmp_path / "strict-v1", None) + entry = plugin / "plugin.py" + entry.write_text( + entry + .read_text() + .replace("import hashlib", "import hashlib\nfrom datetime import date\nfrom enum import Enum") + .replace('ConfigDict(extra="forbid", frozen=True)', 'ConfigDict(extra="forbid", frozen=True, strict=True)') + .replace( + "class Config(BaseModel):", + 'class Mode(str, Enum):\n NORMAL = "normal"\n URGENT = "urgent"\n\n\nclass Config(BaseModel):', + ) + .replace( + ' title_prefix: str = "Incident report"', + ' mode: Mode = Mode.NORMAL\n observed_at: date = date(2026, 10, 10)\n labels: tuple[str, ...] = ("初始",)\n title_prefix: str = "Incident report"', + ) + .replace( + " return IncidentReportPipeline(config)", + ' prefix = f"{config.mode.value}/{config.observed_at.isoformat()}/{config.labels}"\n return IncidentReportPipeline(config.model_copy(update={"title_prefix": prefix}))', + ) + ) + async with running_server(tmp_path) as client: + scope = await create_scope(client, "strict-configuration") + await reload_plugin(client, tmp_path, plugin, 0) + await change(client, "enable", 1, "enabled") + await capture(client, scope, "before-restart") + reports = await contents(client, scope, "incident-report", 1) + assert reports[0]["content"]["title"].startswith("normal/2026-10-10/('初始',):") + async with running_server(tmp_path) as client: + state = await plugin_state(client, "enabled") + await capture(client, scope, "after-restart") + reports = await contents(client, scope, "incident-report", 2) + assert all(item["content"]["title"].startswith("normal/2026-10-10/('初始',):") for item in reports) + replacement = deploy_plugin( + plugin, + tmp_path / "strict-v2", + None, + version="2", + config={"mode": "urgent", "observed_at": "2026-10-11", "labels": ["更新"]}, + ) + await reload_plugin(client, tmp_path, replacement, state["binding_generation"], "enabled") + await capture(client, scope, "after-reload") + reports = await contents(client, scope, "incident-report", 3) + assert next(item for item in reports if item["sources"][0]["source_id"] == "after-reload")["content"][ + "title" + ].startswith("urgent/2026-10-11/('更新',):") + + +def test_plugin_cannot_register_the_builtin_atomic_memory_family(tmp_path: Path) -> None: + asyncio.run(_reject_builtin_atomic_memory_plugin(tmp_path)) + + +async def _reject_builtin_atomic_memory_plugin(tmp_path: Path) -> None: + plugin = deploy_plugin(EXAMPLE, tmp_path / "builtin-family", None) + entry = plugin / "plugin.py" + entry.write_text(entry.read_text().replace('"incident-report"', '"atomic-memory"')) + async with running_server(tmp_path) as client: + publish_plugin(tmp_path, plugin) + response = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 0}) + assert response.status_code == 422, response.text + response = await client.get("/v1/admin/artifact-plugins") + assert response.status_code == 200, response.text + assert response.json()["items"] == [] + + +def test_in_memory_database_rejects_plugin_enable_without_changing_registration(tmp_path: Path) -> None: + asyncio.run(_reject_in_memory_plugin_generation(tmp_path)) + + +async def _reject_in_memory_plugin_generation(tmp_path: Path) -> None: + plugin = deploy_plugin(EXAMPLE, tmp_path / "memory-plugin", None) + publish_plugin(tmp_path, plugin) + async with running_server(tmp_path, database_url="sqlite+aiosqlite:///:memory:") as client: + state = await plugin_state(client, "disabled") + scope = await create_scope(client, "in-memory-input") + await capture(client, scope, "pending-input") + response = await client.post( + f"{REGISTRATION}/enable", json={"expected_generation": state["binding_generation"]} + ) + assert response.status_code == 422, response.text + assert await plugin_state(client, "disabled") == state + item = (await enable_all(client, expected_failures=frozenset({"incident-reports"})))["incident-reports"] + assert item["outcome"] == "failed" + assert item["error"] == "invalid_plugin" + assert item["state"] == state + assert await artifacts(client, scope, 0) == [] + + +def test_plugin_manifests_reject_enable_fields_without_replacing_saved_definition(tmp_path: Path) -> None: + asyncio.run(_reject_manifest_enable_fields(tmp_path)) + + +async def _reject_manifest_enable_fields(tmp_path: Path) -> None: + report = deploy_plugin(EXAMPLE, tmp_path / "report-v1", None, config={"title_prefix": "Saved"}) + publish_plugin(tmp_path, report) + invalid_registrations = [] + for field in ("enable", "enabled"): + binding_name = f"invalid-{field}" + plugin = deploy_plugin( + SUMMARY_EXAMPLE, + tmp_path / binding_name, + None, + binding_name=binding_name, + **{field: True}, + ) + publish_plugin(tmp_path, plugin) + invalid_registrations.append((field, binding_name)) + + async with running_server(tmp_path) as client: + state = await plugin_state(client, "disabled") + response = await client.get("/v1/admin/artifact-plugins") + assert response.status_code == 200, response.text + assert response.json()["items"] == [state] + for field, binding_name in invalid_registrations: + registration = f"/v1/admin/artifact-plugins/{binding_name}" + assert (await client.get(registration)).status_code == 404 + for value in (True, False): + candidate = deploy_plugin( + SUMMARY_EXAMPLE, + tmp_path / f"{binding_name}-{value}", + None, + binding_name=binding_name, + **{field: value}, + ) + publish_plugin(tmp_path, candidate) + rejected = await client.post(f"{registration}/reload", json={"expected_generation": 0}) + assert rejected.status_code == 422, rejected.text + assert (await client.get(registration)).status_code == 404 + + assert set(await enable_all(client)) == {"incident-reports"} + state = await plugin_state(client, "enabled") + scope = await create_scope(client, "manifest-validation") + await capture(client, scope, "before-invalid-update") + await artifacts(client, scope, 1) + for field in ("enable", "enabled"): + for value in (True, False): + candidate = deploy_plugin( + EXAMPLE, + tmp_path / f"report-{field}-{value}", + None, + config={"title_prefix": "Rejected"}, + **{field: value}, + ) + publish_plugin(tmp_path, candidate) + rejected = await client.post( + f"{REGISTRATION}/reload", json={"expected_generation": state["binding_generation"]} + ) + assert rejected.status_code == 422, rejected.text + assert await plugin_state(client, "enabled") == state + await capture(client, scope, "after-invalid-update") + results = await contents(client, scope, "incident-report", 2) + assert all(item["content"]["title"].startswith("Saved:") for item in results) + + +def test_user_plugin_through_server_hot_management_and_restart(tmp_path: Path) -> None: + asyncio.run(_use_plugin_through_server(tmp_path)) + + +async def _use_plugin_through_server(tmp_path: Path) -> None: + # Existing Scopes and inputs are created through the delivered HTTP API. + async with running_server(tmp_path) as client: + scope = await create_scope(client, "reports") + other_scope = await create_scope(client, "other") + await capture(client, scope, "ticket-1") + plugin = deploy_plugin(EXAMPLE, tmp_path / "incident-report-v1", [scope], config={"title_prefix": "Report"}) + factory_signal = tmp_path / "pipeline-created" + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text().replace( + " return IncidentReportPipeline(config)", + f" from pathlib import Path\n Path({str(factory_signal)!r}).write_text('created')\n return IncidentReportPipeline(config)", + ) + ) + publish_plugin(tmp_path, plugin) + broken = deploy_plugin( + SUMMARY_EXAMPLE, tmp_path / "broken-v1", [scope], binding_name="broken-summary", config={"unknown": True} + ) + publish_plugin(tmp_path, broken) + unregistered = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "unregistered-plugin", [scope]) + # Neither version directories outside the root nor nested entries are discovered. + nested = tmp_path / "plugins" / "nested" + nested.mkdir() + (nested / "ticket-summaries").symlink_to(unregistered, target_is_directory=True) + envfile = tmp_path / "server.env" + envfile.write_text(f"POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR={tmp_path / 'plugins'}\n") + async with running_server(tmp_path, envfile=envfile, directory_from_env=False) as client: + state = await plugin_state(client, "disabled") + assert state["binding_generation"] == 1 + registrations = (await client.get("/v1/admin/artifact-plugins")).json()["items"] + assert [item["binding_name"] for item in registrations] == ["incident-reports"] + await asyncio.sleep(2) + assert not factory_signal.exists(), "Loading a declaration must not construct the generation pipeline" + assert await artifacts(client, scope, 0) == [] + metadata = (await client.get("/v1/capabilities")).json() + assert "incident-report" in metadata["artifact_families"] + schema = next(item for item in metadata["custom_artifact_schemas"] if item["family"] == "incident-report") + assert schema["schema_version"] == "1" + assert schema["search"]["indexes"][0]["fields"] == ["title", "evidence_text"] + denied = await client.post("/v1/admin/artifact-plugins/enable-all", headers={"Authorization": ""}) + assert denied.status_code == 401 + obsolete = await client.post( + f"{REGISTRATION}/reload", + json={ + "expected_generation": 1, + "directory": str(plugin), + "scope_ids": [scope], + "config": {}, + "enabled": True, + }, + ) + assert obsolete.status_code == 422, obsolete.text + assert (await client.put(REGISTRATION, json={"expected_generation": 1})).status_code == 405 + accepted = await enable_all(client) + assert accepted["incident-reports"]["outcome"] == "accepted" + state = await plugin_state(client, "enabled") + assert state["binding_generation"] == 2 + await artifacts(client, scope, 1) + assert factory_signal.exists() + assert (await enable_all(client))["incident-reports"]["outcome"] == "skipped" + await capture(client, scope, "ticket-2") + await capture(client, other_scope, "outside-scope") + items = await artifacts(client, scope, 2) + assert {item["revision"] for item in items} == {1} + page = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report", params={"limit": 1}) + assert page.status_code == 200, page.text + next_page = await client.get( + f"/v1/scopes/{scope}/artifacts/incident-report", params={"limit": 1, "cursor": page.json()["next_cursor"]} + ) + assert next_page.status_code == 200, next_page.text + assert {page.json()["items"][0]["artifact_id"], next_page.json()["items"][0]["artifact_id"]} == { + item["artifact_id"] for item in items + } + for item in items: + result = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{item['artifact_id']}") + assert result.status_code == 200, result.text + artifact = result.json() + assert artifact["schema_version"] == "1" + assert artifact["content"]["title"].startswith("Report: ticket-") + assert artifact["content"]["evidence_text"].startswith("Outage ticket-") + assert artifact["sources"] == [ + {"source_type": "content", "source_id": artifact["content"]["title"].split(": ")[1]} + ] + cached = await client.get( + f"/v1/scopes/{scope}/artifacts/incident-report/{item['artifact_id']}", + headers={"If-None-Match": result.headers["etag"]}, + ) + assert cached.status_code == 304 + assert len((await query(client, scope))["results"]) == 2 + assert (await query(client, other_scope))["results"] == [] + await capture(client, scope, "ticket-1") + stale = await client.post(f"{REGISTRATION}/disable", json={"expected_generation": 99}) + assert stale.status_code == 409 + state = await change(client, "disable", 2, "disabled") + await capture(client, scope, "ticket-3") + assert len((await query(client, scope))["results"]) == 2 + assert "enabled" not in yaml.safe_load((plugin / "plugin.yml").read_text()) + replacement = deploy_plugin( + EXAMPLE, tmp_path / "incident-report-v2", [scope], version="2", config={"title_prefix": "Replacement"} + ) + publish_plugin(tmp_path, replacement) + # Environment variables take priority over an envfile. Scanning cannot reset a saved disable or update its code. + envfile.write_text(f"POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR={tmp_path / 'wrong-discovery-root'}\n") + async with running_server(tmp_path, envfile=envfile) as client: + state = await plugin_state(client, "disabled") + assert state["binding_generation"] == 3 + assert len((await query(client, scope))["results"]) == 2 + assert len((await client.get("/v1/admin/artifact-plugins")).json()["items"]) == 1 + state = await change(client, "enable", state["binding_generation"], "enabled") + assert state["binding_generation"] == 4 + results = await contents(client, scope, "incident-report", 3) + assert all(item["content"]["title"].startswith("Report:") for item in results) + # Enabled registrations resume without another batch enable and keep the resolved saved version. + async with running_server(tmp_path) as client: + state = await plugin_state(client, "enabled") + assert state["binding_generation"] > 4 + await capture(client, scope, "ticket-4") + results = await contents(client, scope, "incident-report", 4) + assert all(item["content"]["title"].startswith("Report:") for item in results) + # Explicit reload picks up the switched entry and keeps this plugin enabled. + state = await reload_plugin(client, tmp_path, replacement, state["binding_generation"], "enabled") + unchanged = await reload_plugin(client, tmp_path, replacement, state["binding_generation"], "enabled") + assert unchanged["binding_generation"] == state["binding_generation"] + unloaded = await change(client, "unload", state["binding_generation"], "unloaded") + shutil.rmtree(plugin) + # Startup must not load even a valid discovery entry for an unloaded registration. + async with running_server(tmp_path) as client: + state = await plugin_state(client, "unloaded") + assert state["binding_generation"] == unloaded["binding_generation"] + assert (await enable_all(client))["incident-reports"]["outcome"] == "skipped" + assert len((await query(client, scope))["results"]) == 4 + await capture(client, scope, "ticket-5") + assert len(await artifacts(client, scope, 4)) == 4 + for item in results: + result = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{item['artifact_id']}") + assert result.status_code == 200, result.text + state = await reload_plugin(client, tmp_path, replacement, state["binding_generation"]) + assert len((await query(client, scope))["results"]) == 4 + await change(client, "enable", state["binding_generation"], "enabled") + results = await contents(client, scope, "incident-report", 5) + assert ( + next(item for item in results if item["sources"][0]["source_id"] == "ticket-5")["content"]["title"] + == "Replacement: ticket-5" + ) + + +def test_generation_failure_retries_uncommitted_inputs_after_config_update(tmp_path: Path) -> None: + asyncio.run(_recover_generation_failure(tmp_path)) + + +async def _recover_generation_failure(tmp_path: Path) -> None: + code = tmp_path / "user-plugin-code" + shutil.copytree(EXAMPLE, code, ignore=shutil.ignore_patterns("__pycache__")) + entry = code / "plugin.py" + entry.write_text( + entry + .read_text() + .replace( + ' title_prefix: str = "Incident report"', + ' deployment_label: str = Field(alias="label")\n title_prefix: str = "Incident report"', + ) + .replace( + " return IncidentReportPipeline(config)", + ' if config.deployment_label != "recovery":\n raise ValueError("Invalid deployment label")\n return IncidentReportPipeline(config)', + ) + ) + async with running_server(tmp_path) as client: + scope = await create_scope(client, "reports") + replacement_scope = await create_scope(client, "replacement") + plugin = deploy_plugin( + code, tmp_path / "plugin-v1", [scope], config={"fail_on_text": "Outage", "label": "recovery"} + ) + await capture(client, scope, "failed-ticket") + await reload_plugin(client, tmp_path, plugin, 0) + await change(client, "enable", 1, "enabled") + # The product exposes Worker failures through its authenticated metrics. + await wait_for_worker_metric(client, "failed_total") + assert await artifacts(client, scope, 0) == [] + invalid_directory = deploy_plugin(plugin, tmp_path / "invalid-config", [scope], config={"unknown": True}) + publish_plugin(tmp_path, invalid_directory, "incident-reports") + invalid = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert invalid.status_code == 422, invalid.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + invalid_entry = deploy_plugin(plugin, tmp_path / "invalid-entry", [scope]) + entry = invalid_entry / "plugin.py" + entry.write_text( + entry.read_text().replace( + "async def create_pipeline(ctx: GenerationContext, config: Config)", + "async def create_pipeline(config: Config)", + ) + ) + publish_plugin(tmp_path, invalid_entry, "incident-reports") + invalid = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert invalid.status_code == 422, invalid.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + mismatch = deploy_plugin(plugin, tmp_path / "wrong-binding", [scope], binding_name="another-binding") + publish_plugin(tmp_path, mismatch, "incident-reports") + invalid = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert invalid.status_code == 422, invalid.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + incompatible = deploy_plugin(plugin, tmp_path / "incompatible-schema", [scope]) + entry = incompatible / "plugin.py" + entry.write_text(entry.read_text().replace("Field(min_length=1)", "Field(min_length=2)")) + publish_plugin(tmp_path, incompatible, "incident-reports") + conflict = await client.post(f"{REGISTRATION}/reload", json={"expected_generation": 2}) + assert conflict.status_code == 409, conflict.text + assert (await plugin_state(client, "enabled"))["binding_generation"] == 2 + replacement = deploy_plugin( + plugin, tmp_path / "plugin-v2", [replacement_scope], version="2", config={"label": "recovery"} + ) + await reload_plugin(client, tmp_path, replacement, 2, "enabled") + await capture(client, replacement_scope, "replacement-ticket") + await artifacts(client, replacement_scope, 1) + await wait_for_worker_metric(client, "completed_total") + assert await artifacts(client, scope, 0) == [] + metrics = await client.get("/metrics") + assert metrics.status_code == 200, metrics.text + failures = [ + float(line.rsplit(" ", 1)[1]) + for line in metrics.text.splitlines() + if line.startswith('powercontext_server_artifact_processing_failed_total{family="incident-report"}') + ] + assert failures == [0.0], "Removed Scopes must not keep dispatching failed Workers" + # Re-adding a Scope resumes its retained input instead of losing the failed window. + restored = deploy_plugin( + replacement, tmp_path / "plugin-v3", [scope, replacement_scope], version="3", config={"label": "recovery"} + ) + await reload_plugin(client, tmp_path, restored, 3, "enabled") + items = await artifacts(client, scope, 1) + assert items[0]["revision"] == 1 + # First publication is asynchronous even after generation has committed. + await _wait_for_search_ready(client, scope, "Outage") + assert len((await query(client, scope))["results"]) == 1 + + +def test_disabling_an_active_worker_reclaims_it_before_reenabling(tmp_path: Path) -> None: + asyncio.run(_disable_active_generation(tmp_path)) + + +async def _disable_active_generation(tmp_path: Path) -> None: + code = tmp_path / "user-plugin-code" + shutil.copytree(EXAMPLE, code, ignore=shutil.ignore_patterns("__pycache__")) + entry = code / "plugin.py" + entry.write_text( + entry + .read_text() + .replace("import hashlib", "import asyncio\nimport hashlib") + .replace( + ' title_prefix: str = "Incident report"', + ' block_generation: bool = False\n title_prefix: str = "Incident report"', + ) + + """ + +class SummaryPipeline: + def __init__(self, ctx, config): + self.ctx = ctx + self.config = config + + async def generate(self, batch): + if self.config.block_generation: + await asyncio.Future() + previous = await self.ctx.read_latest("incident-report", "latest-report") + old_text = "" if previous is None else previous.content["evidence_text"] + "\\n" + return [ArtifactDraft( + family="incident-report", schema_version="1", artifact_id="latest-report", + expected_revision=0 if previous is None else previous.ref.revision, + content={"title": self.config.title_prefix, + "evidence_text": old_text + "\\n".join(TextEvidence.model_validate(source.value).content for source in batch.sources)}, + source_refs=tuple(source.ref for source in batch.sources), + artifact_refs=() if previous is None else (previous.ref,), + )] + +async def create_pipeline(ctx, config): + return SummaryPipeline(ctx, config) +""" + ) + async with running_server(tmp_path) as client: + scope = await create_scope(client, "reports") + plugin = deploy_plugin(code, tmp_path / "plugin-v1", [scope], config={"block_generation": True}) + await capture(client, scope, "active-ticket") + await reload_plugin(client, tmp_path, plugin, 0) + await change(client, "enable", 1, "enabled") + await wait_for_worker_metric(client, "used_workers") + await change(client, "disable", 2, "disabled") + assert await artifacts(client, scope, 0) == [] + replacement = deploy_plugin( + plugin, + tmp_path / "plugin-v2", + [scope], + version="2", + config={"block_generation": False, "title_prefix": "Replacement"}, + ) + state = await reload_plugin(client, tmp_path, replacement, 3) + assert await artifacts(client, scope, 0) == [] + await change(client, "enable", state["binding_generation"], "enabled") + items = await artifacts(client, scope, 1) + response = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/{items[0]['artifact_id']}") + assert response.status_code == 200, response.text + assert response.json()["revision"] == 1 + assert response.json()["content"]["title"] == "Replacement" + await capture(client, scope, "update-ticket") + async with asyncio.timeout(45): + while True: + response = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/latest-report") + assert response.status_code == 200, response.text + if response.json()["revision"] == 2: + break + await asyncio.sleep(0.2) + result = response.json() + assert "active-ticket" in result["content"]["evidence_text"] + assert "update-ticket" in result["content"]["evidence_text"] + assert result["sources"] == [{"source_type": "content", "source_id": "update-ticket"}] + assert result["artifacts"] == [{"family": "incident-report", "artifact_id": "latest-report", "revision": 1}] + assert (await query(client, scope))["results"][0]["revision"] == 2 + previous = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/latest-report/revisions/1") + assert previous.status_code == 200, previous.text + assert "update-ticket" not in previous.json()["content"]["evidence_text"] + + +def test_multiple_plugin_manifests_apply_independent_configuration(tmp_path: Path) -> None: + asyncio.run(_use_independent_plugins(tmp_path)) + + +async def _use_independent_plugins(tmp_path: Path) -> None: + summary_registration = "/v1/admin/artifact-plugins/ticket-summaries" + async with running_server(tmp_path) as client: + report_scope = await create_scope(client, "reports") + other_report_scope = await create_scope(client, "other-reports") + summary_scope = await create_scope(client, "summaries") + report = deploy_plugin( + EXAMPLE, + tmp_path / "report-v1", + [report_scope, other_report_scope], + max_sources=1, + processing={"interval_seconds": 0.5, "max_workers": 2, "timeout_seconds": 60}, + config={"title_prefix": "独立报告"}, + ) + # This user pipeline accepts a single evidence item per invocation. + entry = report / "plugin.py" + entry.write_text( + entry.read_text().replace( + " drafts = []", + ' if len(batch.sources) != 1:\n raise ValueError("Expected one ticket per report invocation")\n drafts = []', + ) + ) + summary = deploy_plugin( + SUMMARY_EXAMPLE, + tmp_path / "summary-v1", + [summary_scope], + max_sources=2, + processing={"interval_seconds": 15, "max_workers": 1, "timeout_seconds": 90}, + config={"language": "Français"}, + ) + for scope in (report_scope, summary_scope): + for number in range(3): + await capture(client, scope, f"ticket-{number}") + await capture(client, other_report_scope, "another-ticket") + assert (await client.get("/v1/admin/artifact-plugins")).json()["items"] == [] + for registration, directory in ((REGISTRATION, report), (summary_registration, summary)): + await reload_plugin(client, tmp_path, directory, 0, registration=registration) + assert await artifacts(client, report_scope, 0) == [] + assert await artifacts(client, summary_scope, 0, "ticket-summary") == [] + # A temporarily unavailable deployed version fails independently; successful items stay accepted. + unavailable = summary.with_name("summary-v1-unavailable") + summary.rename(unavailable) + results = await enable_all(client, expected_failures=frozenset({"ticket-summaries"})) + assert results["incident-reports"]["outcome"] == "accepted" + assert results["ticket-summaries"]["outcome"] == "failed" + assert results["ticket-summaries"]["error"] == "invalid_plugin" + assert (await plugin_state(client, "disabled", summary_registration))["binding_generation"] == 1 + await plugin_state(client, "enabled") + summary = unavailable.rename(summary) + results = await enable_all(client) + assert results["incident-reports"]["outcome"] == "skipped" + assert results["ticket-summaries"]["outcome"] == "accepted" + await plugin_state(client, "enabled", summary_registration) + assert all(item["outcome"] == "skipped" for item in (await enable_all(client)).values()) + started = asyncio.get_running_loop().time() + assert await worker_metric(client, "max_workers", "incident-report") == 2 + assert await worker_metric(client, "max_workers", "ticket-summary") == 1 + # No second automatic window may be admitted before this binding's interval. + # The first window can run immediately and must contain at most two Sources. + while asyncio.get_running_loop().time() < started + 14: + response = await client.get(f"/v1/scopes/{summary_scope}/artifacts/ticket-summary") + assert response.status_code == 200, response.text + assert len(response.json()["items"]) <= 1, "The summary binding ignored its 15-second interval" + await asyncio.sleep(0.2) + reports = await contents(client, report_scope, "incident-report", 3) + assert all(item["content"]["title"].startswith("独立报告:") for item in reports) + assert all(len(item["sources"]) == 1 for item in reports) + await artifacts(client, other_report_scope, 1) + summaries = await contents(client, summary_scope, "ticket-summary", 2) + assert sorted(len(item["sources"]) for item in summaries) == [1, 2] + assert all(item["content"]["language"] == "Français" for item in summaries) + assert {source["source_id"] for item in summaries for source in item["sources"]} == { + "ticket-0", + "ticket-1", + "ticket-2", + } + assert (await query(client, report_scope, "ticket-summary"))["results"] == [] + assert (await query(client, summary_scope))["results"] == [] + assert "enabled" not in yaml.safe_load((report / "plugin.yml").read_text()) + assert "enabled" not in yaml.safe_load((summary / "plugin.yml").read_text()) + replacement = deploy_plugin( + summary, + tmp_path / "summary-v2", + [summary_scope], + version="2", + max_sources=1, + processing={"interval_seconds": 0.5, "max_workers": 2, "timeout_seconds": 60}, + config={"language": "Deutsch"}, + ) + await reload_plugin(client, tmp_path, replacement, 2, "enabled", summary_registration) + for number in (3, 4): + await capture(client, summary_scope, f"ticket-{number}") + summaries = await contents(client, summary_scope, "ticket-summary", 4) + updated = [item for item in summaries if item["content"]["language"] == "Deutsch"] + assert len(updated) == 2 + assert all(len(item["sources"]) == 1 for item in updated) + assert await worker_metric(client, "max_workers", "ticket-summary") == 2 + assert await worker_metric(client, "max_workers", "incident-report") == 2 + await change(client, "unload", 3, "unloaded", summary_registration) + await capture(client, report_scope, "report-after-summary-unload") + await artifacts(client, report_scope, 4) + assert len((await query(client, summary_scope, "ticket-summary"))["results"]) == 4 + + +def test_worker_timeout_is_local_to_its_plugin_and_code_update_recovers(tmp_path: Path) -> None: + asyncio.run(_recover_timed_out_plugin(tmp_path)) + + +async def _recover_timed_out_plugin(tmp_path: Path) -> None: + summary_registration = "/v1/admin/artifact-plugins/ticket-summaries" + async with running_server(tmp_path) as client: + report_scope = await create_scope(client, "timed-out-reports") + summary_scope = await create_scope(client, "healthy-summaries") + blocked = deploy_plugin( + EXAMPLE, + tmp_path / "blocked-v1", + [report_scope], + processing={"interval_seconds": 1, "max_workers": 1, "timeout_seconds": 5}, + ) + entry = blocked / "plugin.py" + entry.write_text( + entry + .read_text() + .replace("import hashlib", "import asyncio\nimport hashlib") + .replace(" drafts = []", " await asyncio.Future()\n drafts = []") + ) + healthy = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "healthy-v1", [summary_scope]) + for registration, directory in ((REGISTRATION, blocked), (summary_registration, healthy)): + await reload_plugin(client, tmp_path, directory, 0, registration=registration) + assert all(item["outcome"] == "accepted" for item in (await enable_all(client)).values()) + await plugin_state(client, "enabled") + await plugin_state(client, "enabled", summary_registration) + await capture(client, report_scope, "timed-out-ticket") + await capture(client, summary_scope, "healthy-ticket") + await wait_for_worker_metric(client, "timeouts_total") + assert await artifacts(client, report_scope, 0) == [] + healthy_result = await contents(client, summary_scope, "ticket-summary", 1) + assert healthy_result[0]["content"]["source_ids"] == ["healthy-ticket"] + assert (await plugin_state(client, "enabled", summary_registration))["binding_generation"] == 2 + recovered = deploy_plugin( + EXAMPLE, + tmp_path / "report-v2", + [report_scope], + version="2", + config={"title_prefix": "Recovered"}, + ) + await reload_plugin(client, tmp_path, recovered, 2, "enabled") + result = await contents(client, report_scope, "incident-report", 1) + assert result[0]["revision"] == 1 + assert result[0]["content"]["title"] == "Recovered: timed-out-ticket" + assert len((await query(client, summary_scope, "ticket-summary"))["results"]) == 1 + + +def test_enable_all_fixes_its_snapshot_and_reports_concurrent_changes(tmp_path: Path) -> None: + asyncio.run(_enable_all_with_concurrent_changes(tmp_path)) + + +async def _enable_all_with_concurrent_changes(tmp_path: Path) -> None: + async with running_server(tmp_path) as client: + scope = await create_scope(client, "batch-management") + report = deploy_plugin(EXAMPLE, tmp_path / "report-v1", [scope]) + gate = tmp_path / "validation-paused" + entered = tmp_path / "validation-entered" + entry = report / "plugin.py" + # A user module with a slow import allows independent HTTP management during validation. + entry.write_text( + entry.read_text() + + f""" + +from pathlib import Path +import time +if Path({str(gate)!r}).exists(): + Path({str(entered)!r}).write_text("entered") + while Path({str(gate)!r}).exists(): + time.sleep(0.05) +""" + ) + await reload_plugin(client, tmp_path, report, 0) + summary = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "summary-v1", [scope]) + publish_plugin(tmp_path, summary) + # Merely publishing an entry after startup does not make it eligible for a batch enable. + assert (await client.get(SUMMARY_REGISTRATION)).status_code == 404 + gate.touch() + batch = asyncio.create_task(enable_all(client, expected_failures=frozenset({"incident-reports"}))) + try: + async with asyncio.timeout(10): + while True: + if entered.exists(): + break + await asyncio.sleep(0.05) + await reload_plugin(client, tmp_path, summary, 0, registration=SUMMARY_REGISTRATION) + replacement = deploy_plugin(EXAMPLE, tmp_path / "report-v2", [scope], version="2") + updated = await reload_plugin(client, tmp_path, replacement, 1) + assert updated["binding_generation"] == 2 + finally: + gate.unlink(missing_ok=True) + results = await batch + assert set(results) == {"incident-reports"}, "A batch must not include plugins loaded after its snapshot" + assert results["incident-reports"]["outcome"] == "failed" + assert results["incident-reports"]["error"] == "conflict" + assert results["incident-reports"]["state"]["binding_generation"] == 2 + assert (await plugin_state(client, "disabled"))["binding_generation"] == 2 + assert (await plugin_state(client, "disabled", SUMMARY_REGISTRATION))["binding_generation"] == 1 + assert all(item["outcome"] == "accepted" for item in (await enable_all(client)).values()) + # Transitions and completed enables are both ineligible for an immediate repeat. + assert all(item["outcome"] == "skipped" for item in (await enable_all(client)).values()) + await plugin_state(client, "enabled") + await plugin_state(client, "enabled", SUMMARY_REGISTRATION) + + +def test_failed_restore_waits_for_explicit_reload_and_preserves_progress(tmp_path: Path) -> None: + asyncio.run(_repair_failed_restore(tmp_path)) + + +async def _repair_failed_restore(tmp_path: Path) -> None: + async with running_server(tmp_path) as client: + scope = await create_scope(client, "failed-restore") + healthy_scope = await create_scope(client, "healthy") + report = deploy_plugin(EXAMPLE, tmp_path / "report-v1", [scope]) + summary = deploy_plugin(SUMMARY_EXAMPLE, tmp_path / "summary-v1", [healthy_scope]) + await reload_plugin(client, tmp_path, report, 0) + await change(client, "enable", 1, "enabled") + await reload_plugin(client, tmp_path, summary, 0, registration=SUMMARY_REGISTRATION) + await capture(client, scope, "before-failure") + await artifacts(client, scope, 1) + # An unavailable saved version must fail restoration even if the discovery entry points at a valid update. + report.rename(tmp_path / "report-v1-unavailable") + replacement = deploy_plugin( + EXAMPLE, tmp_path / "report-v2", [scope], version="2", config={"title_prefix": "Repaired"} + ) + publish_plugin(tmp_path, replacement) + async with running_server(tmp_path) as client: + failed = await plugin_state(client, "failed") + assert failed["binding_generation"] == 3 + assert failed["last_error"] + await capture(client, scope, "during-failure") + results = await enable_all(client) + assert results["incident-reports"]["outcome"] == "skipped" + assert results["ticket-summaries"]["outcome"] == "accepted" + await plugin_state(client, "enabled", SUMMARY_REGISTRATION) + await capture(client, healthy_scope, "healthy-ticket") + await artifacts(client, healthy_scope, 1, "ticket-summary") + assert len((await query(client, scope))["results"]) == 1 + # Failed is durable, never auto-enabled by startup or by enable-all. + async with running_server(tmp_path) as client: + state = await plugin_state(client, "failed") + assert state["binding_generation"] == failed["binding_generation"] + assert all(item["outcome"] == "skipped" for item in (await enable_all(client)).values()) + invalid = await client.post(f"{REGISTRATION}/enable", json={"expected_generation": state["binding_generation"]}) + assert invalid.status_code == 422, invalid.text + state = await reload_plugin(client, tmp_path, replacement, state["binding_generation"]) + assert await artifacts(client, scope, 1) + assert (await enable_all(client))["incident-reports"]["outcome"] == "accepted" + await plugin_state(client, "enabled") + results = await contents(client, scope, "incident-report", 2) + repaired = next(item for item in results if item["sources"][0]["source_id"] == "during-failure") + assert repaired["revision"] == 1 + assert repaired["content"]["title"] == "Repaired: during-failure" + assert len((await query(client, healthy_scope, "ticket-summary"))["results"]) == 1 diff --git a/tests/e2e/test_artifact_vector_http.py b/tests/e2e/test_artifact_vector_http.py new file mode 100644 index 0000000000..e01b7de79d --- /dev/null +++ b/tests/e2e/test_artifact_vector_http.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Installed Server and spawned generation Worker share text/vector/hybrid search.""" + +import asyncio + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from tests.builtin.persistence.test_artifact_indexes import PROFILE, body +from tests.e2e.test_artifact_plugin_search import _PIPELINE, _write_report +from tests.e2e.test_artifact_plugins import ( + EXAMPLE, + create_scope, + deploy_plugin, + enable_all, + reload_plugin, + running_server, +) + +_DECLARATION = """ +from powercontext.artifact_plugins import ArtifactSearchDeclaration, TextIndex, VectorIndex, FragmentProjection +from powercontext.builtin.artifacts.memory import EmbeddingProfile + +class Feature(BaseModel): + id: str + vector: list[float] + profile: EmbeddingProfile + +class SearchReport(IncidentReport): + private_text: str + fragments: list[Feature] + +families = (ArtifactFamily( + family="incident-report", schema_version="1", content_model=SearchReport, + search=ArtifactSearchDeclaration( + version="1", modes=("text", "vector", "hybrid"), default_mode="text", + indexes=( + TextIndex(name="lexical", fields=("title", "evidence_text"), projection_version="1"), + VectorIndex(name="visual", projection_version="1", source="content", + profile=EmbeddingProfile(profile_id="features-v1", model="fixture", dimension=2, normalization="none"), + vector_field="vector", profile_field="profile", + fragments=FragmentProjection(array_field="fragments", id_field="id")), + ), + ), +),) +inputs = InputContract(projections=inputs.projections, artifact_families=("experience", "skill", "profile")) +""" + + +def test_vector_search_through_http_and_spawned_worker(migration_backend): + config, directory = migration_backend.config, migration_backend.directory + database = config.model_dump(mode="json") + if isinstance(config, OceanBaseConfig): + database["url"] = config.url.get_secret_value() + + async def exercise(): + async with running_server(directory, database_settings=database) as client: + scope = await create_scope(client, "vector-search") + plugin = deploy_plugin(EXAMPLE, directory / "vector-plugin", [scope]) + entry = plugin / "plugin.py" + pipeline = _PIPELINE.replace( + " drafts = []", + """ # These use the built-in Searchers, not the custom-family declaration parser. + assert not (await self.ctx.search_artifacts("experience", "Outage")).matches + assert not (await self.ctx.search_artifacts("skill", "Outage")).matches + found = await self.ctx.search_artifacts("incident-report", "Report") + assert all("evidence_text" in artifact.content.model_dump() for artifact in found.artifacts) + for artifact in found.artifacts: + await self.ctx.read_revision(artifact.as_ref()) + drafts = []""", + ).replace( + "content=value, source_refs=(source.ref,),", + "content=value, source_refs=(source.ref,), artifact_refs=tuple(artifact.as_ref() for artifact in found.artifacts),", + ) + entry.write_text(entry.read_text() + _DECLARATION + pipeline, encoding="utf-8") + await reload_plugin(client, directory, plugin, 0) + await enable_all(client) + first = await _write_report( + client, scope, "first", "a", "数据库 timeout", fragments=body("", *([[0, 0]] * 8))["fragments"] + ) + second = await _write_report( + client, scope, "second", "b", "数据库 normal", fragments=body("", [1, 0])["fragments"] + ) + assert second["artifacts"] == [{"family": "incident-report", "artifact_id": "a", "revision": 1}] + endpoint = f"/v1/scopes/{scope}/artifacts/incident-report/search" + vector = {"profile": PROFILE.model_dump(mode="json"), "vector": [0, 0]} + request = {"mode": "vector", "query_vector": vector, "limit": 2, "include_scores": True} + for mode in ("vector", "hybrid"): + response = await client.post( + endpoint, json={**request, "mode": mode, **({"query": "数据库"} if mode == "hybrid" else {})} + ) + assert response.status_code == 200, response.text + results = response.json()["results"] + assert [result["artifact_id"] for result in results] == ["a", "b"] + assert results[0]["content"] == first["content"] + assert set(results[0]) == {"family", "artifact_id", "revision", "content", "lineage", "scores"} + for invalid in ( + {"query": "Report", "indexes": {"vector": ["visual"]}}, + {"query": "Report", "indexes": {"text": []}}, + {**request, "query_vector": {**vector, "vector": [1]}}, + { + **request, + "mode": "hybrid", + "query": "Report", + "fusion": {"method": "rrf", "params": {"weights": {"text": 0, "vector": 0}}}, + }, + ): + response = await client.post(endpoint, json=invalid) + assert response.status_code == 422, response.text + unavailable = await client.post(endpoint, json={"mode": "vector", "query": "Report"}) + assert unavailable.status_code == 503 + denied = await client.post(endpoint, json=request, headers={"Authorization": ""}) + assert denied.status_code == 401 + await _write_report(client, scope, "update", "a", "recovered", revision=2, fragments=[]) + response = await client.post(endpoint, json=request) + assert [item["artifact_id"] for item in response.json()["results"]] == ["b"] + historical = await client.get(f"/v1/scopes/{scope}/artifacts/incident-report/a/revisions/1") + assert historical.json()["content"] == first["content"] + + asyncio.run(exercise()) diff --git a/tests/e2e/test_atomic_memory_description_upgrade.py b/tests/e2e/test_atomic_memory_description_upgrade.py index 615fc2b716..4387ae9ae7 100644 --- a/tests/e2e/test_atomic_memory_description_upgrade.py +++ b/tests/e2e/test_atomic_memory_description_upgrade.py @@ -85,7 +85,7 @@ def test_nonempty_released_memory_upgrades_without_rewriting_history(tmp_path, r ) result = _apply(runner) - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_4_1_006" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_001" verified = runner.verify() assert verified.state == "ready" and verified.readiness_scope == "server" assert not _apply(runner).changed diff --git a/tests/e2e/test_source_artifact_plugins.py b/tests/e2e/test_source_artifact_plugins.py new file mode 100644 index 0000000000..2a29f35f0c --- /dev/null +++ b/tests/e2e/test_source_artifact_plugins.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Source plugin snapshot flows through a real Artifact Worker to searchable evidence.""" + +import asyncio +import shutil + +from fastmcp import Client + +from tests.e2e.test_artifact_plugins import EXAMPLE, artifacts, change, deploy_plugin, reload_plugin +from tests.source_helpers import ROOT, TOKENS, call, create_scope, grant, load_write_cli, read, running_server + + +def test_source_plugin_to_artifact_search_and_exact_read(tmp_path): + asyncio.run(_source_to_artifact(tmp_path)) + + +async def _source_to_artifact(directory): + source_plugin = directory / "tickets" + shutil.copytree( + ROOT / "examples/source_plugins/tickets", source_plugin, ignore=shutil.ignore_patterns("__pycache__") + ) + report_plugin = deploy_plugin(EXAMPLE, directory / "reports", None) + entry = report_plugin / "plugin.py" + # Use the delivered generator with the Source plugin's persisted text projection. + entry.write_text( + entry.read_text().replace( + '"content": TEXT_EVIDENCE_PROJECTION_KEY', '"example-ticket": TEXT_EVIDENCE_PROJECTION_KEY' + ) + ) + value = { + "namespace": "support", + "project": "powercontext", + "ticket_id": "PC-123", + "title": "数据库恢复", + "body": "Outage resolved after database retry.", + } + environment = {"POWERCONTEXT_SERVER_ARTIFACT_PLUGINS_DIR": str(directory / "plugins")} + (directory / "plugins").mkdir() + async with running_server(directory, environment=environment) as (admin, url, _process): + scope = await create_scope(admin, "source-artifact-chain") + await grant(admin, scope, "deployer") + await grant(admin, scope) + await load_write_cli(source_plugin / "write.yml", url) + await reload_plugin(admin, directory, report_plugin, 0) + await change(admin, "enable", 1, "enabled") + async with Client(url + "/mcp", auth=TOKENS["host"]) as host: + captured = await call( + host, + "pc_source_add", + { + "scope_id": scope, + "capability": "ticket.write", + "capability_version": "1", + "input": value, + }, + ) + source_ref = captured["result"]["source_refs"][0] + assert source_ref["source_type"] == "example-ticket" + assert (await read(admin, scope, source_ref))["content"]["payload"]["capture"]["body"] == value["body"] + [artifact] = await artifacts(admin, scope, 1) + async with asyncio.timeout(30): + while True: + response = await admin.post( + f"/v1/scopes/{scope}/artifacts/incident-report/search", json={"query": "数据库恢复"} + ) + if response.status_code == 200 and response.json()["results"]: + break + assert response.status_code in (200, 503), response.text + await asyncio.sleep(0.1) + [hit] = response.json()["results"] + assert hit["artifact_id"] == artifact["artifact_id"] + assert hit["revision"] == artifact["revision"] + exact = await admin.get( + f"/v1/scopes/{scope}/artifacts/incident-report/{hit['artifact_id']}/revisions/{hit['revision']}" + ) + assert exact.status_code == 200, exact.text + assert exact.json()["content"]["evidence_text"] == value["title"] + "\n\n" + value["body"] + assert exact.json()["sources"] == [source_ref] diff --git a/tests/native/test_oceanbase_engine_recovery.py b/tests/native/test_oceanbase_engine_recovery.py index 298d731c1a..12b189c72b 100644 --- a/tests/native/test_oceanbase_engine_recovery.py +++ b/tests/native/test_oceanbase_engine_recovery.py @@ -175,7 +175,7 @@ def ping(connection, _identity, _guard): _apply(_full_server_runner(backend), policy="manual") resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.revision == "v1_4_1_006" and result.readiness_scope == "server" + assert result.state == "ready" and result.revision == "v1_5_0_001" and result.readiness_scope == "server" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journal.read_text()) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index f1effba04b..75a2517134 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -401,7 +401,7 @@ def test_native_public_migration_switches_an_installed_130_service( if failure in {"startup", "switch"}: assert applied.exit_code == 1, applied.output assert result["error"] == "service_start_failed" - assert result["database_state"] == "ready" and result["revision"] == "v1_4_1_006" + assert result["database_state"] == "ready" and result["revision"] == "v1_5_0_001" assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} phase = "switching" if failure == "switch" else "start_failed" assert json.loads(controller.maintenance_path.read_text())["phase"] == phase diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py index 92e89a677f..9292fbf5fe 100644 --- a/tests/test_api_contract.py +++ b/tests/test_api_contract.py @@ -328,6 +328,7 @@ def test_capabilities_report_semantics_without_runtime_tuning_values() -> None: assert set(properties) == { "artifact_dreaming", "artifact_dreaming_operations", + "custom_artifact_schemas", "source_types", "artifact_families", "memory_extraction", @@ -1010,7 +1011,8 @@ def test_standard_artifact_reads_share_topic_memory_family_and_display_metadata( for path in read_paths: family = next(parameter for parameter in paths[path]["get"]["parameters"] if parameter["name"] == "family") assert family["schema"] == {"$ref": "#/components/schemas/ArtifactReadFamily"} - assert "topic-memory" in contract["components"]["schemas"]["ArtifactReadFamily"]["enum"] + http_models.ArtifactReadFamily.model_validate("topic-memory") + http_models.ArtifactReadFamily.model_validate("incident-report") assert "topic-memory" in contract["components"]["schemas"]["BaseArtifactFamily"]["enum"] item = http_models.ArtifactCollectionItem.model_validate({ "scope_id": "scope-a", @@ -1025,11 +1027,30 @@ def test_standard_artifact_reads_share_topic_memory_family_and_display_metadata( "published_at": "2026-09-10T00:00:00Z", "source_count": 2, }) - assert item.family.value == "topic-memory" + assert item.family == "topic-memory" assert item.title == "A topic" assert item.source_count == 2 +@pytest.mark.parametrize("model", [http_models.ArtifactCollectionItem, ArtifactRevision]) +@pytest.mark.parametrize("family", ["atomic-memory", "incident-report"]) +def test_artifact_read_family_remains_a_string_with_contract_validation(model, family): + value = { + "scope_id": "scope", + "family": family, + "artifact_id": "artifact", + "revision": 1, + "content": {}, + "sources": [], + "artifacts": [], + "content_digest": f"sha256:{'0' * 64}", + } + assert model.model_validate(value).family == family + for invalid in ("", "INVALID", "x" * 129, 123): + with pytest.raises(ValidationError): + model.model_validate({**value, "family": invalid}) + + def test_base_access_uses_a_dedicated_source_type_reference() -> None: contract = yaml.safe_load(CONTRACT_PATH.read_text()) schemas = contract["components"]["schemas"] diff --git a/tests/test_artifact_index_maintenance.py b/tests/test_artifact_index_maintenance.py new file mode 100644 index 0000000000..eb34552420 --- /dev/null +++ b/tests/test_artifact_index_maintenance.py @@ -0,0 +1,266 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Idle index maintenance must not require SQLite's writer lock or delay startup.""" + +import asyncio + +import pytest +from sqlalchemy import insert, select + +from powercontext.artifacts.indexes import ArtifactSearchDeclaration, TextIndex, VectorIndex +from powercontext.artifacts.search import ArtifactSearchUnavailable, ArtifactSearchUnsupported +from powercontext.builtin.inference.models import EmbeddingResult +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ARTIFACT_SEARCH_STATES_TABLE, SCOPES_TABLE +from powercontext.server.artifact_plugins import ArtifactPluginManager +from tests.builtin.persistence.test_artifact_indexes import PROFILE, register, service, store, write + + +@pytest.mark.parametrize("after_rebuild", [False, True]) +def test_unchanged_maintenance_does_not_wait_for_an_unrelated_writer(tmp_path, after_rebuild): + async def run(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'store.db'}", busy_timeout_ms=50) + async with store(config) as (db, builder, publisher): + declaration = ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="text", fields=("text",), projection_version="1"),), + modes=("text",), + default_mode="text", + ) + await register(db, "reports", declaration) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库故障"}) + await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + if after_rebuild: + await builder.refresh("scope", "reports", rebuild=True) + assert await builder.build("scope", "reports") + # A separate connection holds SQLite's writer lock. Idle maintenance + # must still complete using WAL reads, without a busy timeout failure. + async with db.engine.connect() as blocker: + await blocker.exec_driver_sql("BEGIN IMMEDIATE") + try: + assert await builder.refresh("scope", "reports") + assert await builder.build("scope", "reports") + finally: + await blocker.rollback() + assert (await service(db, "reports").search("scope", "reports", {"query": "故障"})).matches + + asyncio.run(run()) + + +def test_manager_startup_does_not_wait_for_existing_data_encoding(): + async def run(): + started, release = asyncio.Event(), asyncio.Event() + + class Encoder: + profile = PROFILE + + async def embed(self, texts): + started.set() + await release.wait() + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + async with store() as (db, _, publisher): + await register( + db, + "startup_reports", + ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", fields=("text",), projection_version="1", source="text", profile=PROFILE + ), + ), + modes=("vector",), + default_mode="vector", + ), + ) + async with db.transaction() as connection: + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id="scope", + title="", + summary="", + scope_id_search="scope", + title_search="", + summary_search="", + version=1, + ) + ) + await write(db, publisher, "scope", "startup_reports", "a", {"text": "数据库故障"}) + manager = ArtifactPluginManager( + database=db, + config=SQLiteConfig(), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + embedding_model=Encoder(), + ) + try: + async with asyncio.timeout(5): + await manager.__aenter__() + await started.wait() + release.set() + async with asyncio.timeout(5): + while True: + try: + result = await service(db, "startup_reports").search( + "scope", + "startup_reports", + {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1.0, 0.0]}}, + ) + assert result.matches + break + except ArtifactSearchUnavailable: + await asyncio.sleep(0.01) + finally: + release.set() + await manager.__aexit__() + + asyncio.run(run()) + + +def test_manager_discovers_first_write_without_creating_empty_scope_states(): + async def run(): + async with store() as (db, _, publisher): + await register( + db, + "reports", + ArtifactSearchDeclaration( + version="1", + indexes=(TextIndex(name="text", fields=("text",), projection_version="1"),), + modes=("text",), + default_mode="text", + ), + ) + async with db.transaction() as connection: + for scope in ("active", "empty"): + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id=scope, + title="", + summary="", + scope_id_search=scope, + title_search="", + summary_search="", + version=1, + ) + ) + async with ArtifactPluginManager( + database=db, + config=SQLiteConfig(), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + ): + # Allow startup discovery to finish before the first content arrives. + await asyncio.sleep(0.1) + assert not (await service(db, "reports").search("empty", "reports", {"query": "故障"})).matches + with pytest.raises(ArtifactSearchUnsupported): + await service(db, "reports").search( + "empty", "reports", {"query": "故障", "indexes": {"text": ["unknown"]}} + ) + await write(db, publisher, "active", "reports", "a", {"text": "数据库故障"}) + async with asyncio.timeout(6): + while True: + try: + result = await service(db, "reports").search("active", "reports", {"query": "故障"}) + assert result.matches + break + except ArtifactSearchUnavailable: + await asyncio.sleep(0.05) + async with db.transaction() as connection: + scopes = (await connection.execute(select(ARTIFACT_SEARCH_STATES_TABLE.c.scope_id))).scalars().all() + assert scopes == ["active"] + + asyncio.run(run()) + + +def test_manager_recovers_failed_build_and_cleans_before_rebuilding(): + async def run(): + from powercontext.builtin.persistence.artifact_indexes import COVERAGE, STATES + + failed = asyncio.Event() + + class Encoder: + profile = PROFILE + available = False + + async def embed(self, texts): + if not self.available: + failed.set() + raise RuntimeError("encoder unavailable") # noqa: TRY003 + return EmbeddingResult(vectors=tuple((1.0, 0.0) for _ in texts)) + + encoder = Encoder() + async with store(model=encoder) as (db, builder, publisher): + await register( + db, + "reports", + ArtifactSearchDeclaration( + version="1", + indexes=( + VectorIndex( + name="semantic", fields=("text",), projection_version="1", source="text", profile=PROFILE + ), + ), + modes=("vector",), + default_mode="vector", + ), + ) + await write(db, publisher, "scope", "reports", "a", {"text": "数据库故障"}) + + def manager(): + return ArtifactPluginManager( + database=db, + config=SQLiteConfig(), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + embedding_model=encoder, + ) + + async with manager(): + async with asyncio.timeout(5): + await failed.wait() + encoder.available = True + async with manager(): + async with asyncio.timeout(6): + while True: + try: + result = await service(db, "reports").search( + "scope", + "reports", + {"query_vector": {"profile": PROFILE.model_dump(mode="json"), "vector": [1, 0]}}, + ) + assert result.matches + break + except ArtifactSearchUnavailable: + await asyncio.sleep(0.05) + async with db.transaction() as connection: + old = await connection.scalar(select(STATES.c.build_id).where(STATES.c.state == "ready")) + await builder.refresh("scope", "reports", rebuild=True) + async with asyncio.timeout(6): + while True: + async with db.transaction() as connection: + state = await connection.scalar(select(STATES.c.state).where(STATES.c.build_id != old)) + remains = await connection.scalar( + select(COVERAGE.c.artifact_id).where(COVERAGE.c.build_id == old) + ) + if state == "ready" and remains is None: + break + await asyncio.sleep(0.05) + + asyncio.run(run()) diff --git a/tests/test_artifact_plugin_leases.py b/tests/test_artifact_plugin_leases.py new file mode 100644 index 0000000000..ebb5104132 --- /dev/null +++ b/tests/test_artifact_plugin_leases.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Plugin activation preserves the renewable Supervisor ownership contract.""" + +import asyncio + +from pydantic import SecretStr + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.server.artifact_plugins import ArtifactPluginManager +from tests.e2e.test_artifact_plugins import EXAMPLE, deploy_plugin + + +def test_first_plugin_activation_preserves_an_unexpired_oceanbase_owner(tmp_path): + deploy_plugin(EXAMPLE, tmp_path / "plugins" / "incident-reports", None) + + async def run(): + # SQLite hosts real lease state; OceanBaseConfig exercises configuration + # selection without claiming acceptance against an OceanBase server. + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as storage: + database = storage.database + leases = ArtifactProcessingLeaseRepository() + async with database.transaction() as connection: + owner = await leases.try_acquire(connection, "existing-background-owner", 3600) + assert owner is not None and owner.lease_expires_at is not None + fence = owner.fence("oceanbase") + async with ArtifactPluginManager( + database=database, + config=OceanBaseConfig(url=SecretStr("mysql+aoceanbase://test@localhost:2881/test?charset=utf8mb4")), + supervisor=None, + security_builder=lambda: None, + on_supervisor=lambda _: None, + plugins_dir=tmp_path / "plugins", + ) as manager: + registration = await manager.get("incident-reports") + assert registration.phase == "disabled" + await manager.change("incident-reports", registration.binding_generation, "enabled") + async with asyncio.timeout(10): + while (state := await manager.get("incident-reports")).phase == "activating": # noqa: ASYNC110 + await asyncio.sleep(0.01) + assert state.phase == "enabled", state.last_error + async with database.transaction() as connection: + assert await leases.require_fence(connection, fence) == owner + # Closing the standby must not release another Supervisor's term. + async with database.transaction() as connection: + assert await leases.require_fence(connection, fence) == owner + + asyncio.run(run()) diff --git a/tests/test_artifact_plugin_validation.py b/tests/test_artifact_plugin_validation.py new file mode 100644 index 0000000000..5ac51ef99d --- /dev/null +++ b/tests/test_artifact_plugin_validation.py @@ -0,0 +1,88 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declaration validation in a real child within its existing startup budget.""" + +import asyncio +import os + +import httpx +import pytest + +from powercontext.builtin.artifact_plugin_validation import validate_plugin +from tests.e2e.test_artifact_plugins import EXAMPLE, REGISTRATION, deploy_plugin, enable_all + + +def test_declaration_validation_does_not_spend_its_budget_starting_the_runtime(tmp_path, monkeypatch): + plugin = deploy_plugin(EXAMPLE, tmp_path / "plugin", None, config={}) + entry = plugin / "plugin.py" + entry.write_text( + entry.read_text() + + '\nimport os\nos.environ["PC_TEST_PLUGIN_IMPORTED"] = "child"\n' + + 'print("plugin import output is not the declaration protocol")\n' + ) + monkeypatch.delenv("PC_TEST_PLUGIN_IMPORTED", raising=False) + # A slow runtime bootstrap must not consume declaration validation's 15-second budget. + # This hook is installed only in new Python processes, not in the pytest process. + hook = tmp_path / "startup" + hook.mkdir() + (hook / "sitecustomize.py").write_text( + "import sys, time\n" + "class SlowRuntime:\n" + " def find_spec(self, fullname, path=None, target=None):\n" + " if fullname == 'powercontext.builtin.runtime':\n" + " time.sleep(20)\n" + "sys.meta_path.insert(0, SlowRuntime())\n" + ) + monkeypatch.setenv("PYTHONPATH", os.pathsep.join(filter(None, (str(hook), os.environ.get("PYTHONPATH"))))) + + async def run(): + described = await validate_plugin(str(plugin)) + restored = await validate_plugin(str(plugin), described["manifest"]) + assert "PC_TEST_PLUGIN_IMPORTED" not in os.environ + assert restored == described + assert described["declaration"]["family"] == "incident-report" + assert described["manifest"]["config"]["title_prefix"] == "Incident report" + + asyncio.run(run()) + + +def test_invalid_declaration_is_rejected_by_the_validation_child(tmp_path): + plugin = deploy_plugin(EXAMPLE, tmp_path / "invalid", None, config={"unknown": True}) + + async def run(): + with pytest.raises(ValueError, match="plugin declaration or configuration validation failed"): + await validate_plugin(str(plugin)) + + asyncio.run(run()) + + +def test_enable_all_reports_unexpected_item_failure_immediately(): + item = { + "binding_name": "incident-reports", + "outcome": "failed", + "status_url": REGISTRATION, + "state": {"phase": "disabled"}, + "error": "invalid_plugin", + } + + async def run(): + transport = httpx.MockTransport(lambda request: httpx.Response(200, json={"items": [item]})) + async with httpx.AsyncClient(base_url="http://test", transport=transport) as client: + with pytest.raises(AssertionError, match="invalid_plugin"): + await enable_all(client) + results = await enable_all(client, expected_failures=frozenset({"incident-reports"})) + assert results["incident-reports"] == item + + asyncio.run(run()) diff --git a/tests/test_desktop_contract.py b/tests/test_desktop_contract.py index bbd5564736..f0d031a380 100644 --- a/tests/test_desktop_contract.py +++ b/tests/test_desktop_contract.py @@ -46,3 +46,14 @@ def test_desktop_contract_identity_matches_openapi(manifest: str) -> None: actual = match.group(1) assert actual == expected, f"Stale Desktop contract identity in {manifest}; run 'pnpm --dir desktop generate'." + + +def test_bundled_desktop_contract_has_a_qualified_business_profile() -> None: + native = REPOSITORY_ROOT / "desktop/src-tauri/src" + contract = json.loads((native / "transport/operations.json").read_text(encoding="utf-8")) + profiles = json.loads((native / "connections/compatibility.json").read_text(encoding="utf-8")) + required = {"remember_memory", "search_memory", "get_artifact_revision"} + assert any( + profile["contractSha256"] == contract["contractSha256"] and required <= set(profile["operations"]) + for profile in profiles + ), "The bundled Desktop contract has no qualified profile for save, search and exact reads." diff --git a/tests/test_managed_atomic_maintenance.py b/tests/test_managed_atomic_maintenance.py index 473d68eabe..41f968cbbd 100644 --- a/tests/test_managed_atomic_maintenance.py +++ b/tests/test_managed_atomic_maintenance.py @@ -76,7 +76,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b assert inspected.exit_code == 0, inspected.output plan = json.loads(inspected.output) assert plan["source_revision"] == "s1300" - assert plan["target_revision"] == "v1_4_1_006" + assert plan["target_revision"] == "v1_5_0_001" assert plan["readiness_scope"] == "server" assert historical.read_bytes() == original @@ -103,7 +103,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b with closing(sqlite3.connect(result["backup_ref"])) as connection: assert tuple(connection.iterdump()) == original_dump with closing(sqlite3.connect(historical)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_4_1_006",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] assert connection.execute("SELECT content FROM pc_artifacts").fetchall() == [(b'{"history":"unchanged"}',)] verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) assert verified.exit_code == 0, verified.output diff --git a/tests/test_managed_processing_maintenance.py b/tests/test_managed_processing_maintenance.py index 806d1555b9..b803b5ada9 100644 --- a/tests/test_managed_processing_maintenance.py +++ b/tests/test_managed_processing_maintenance.py @@ -75,7 +75,7 @@ def test_registered_mode_switch_backs_up_preserves_revision_and_fences_old_worke assert result.backup_state == "completed" assert result.backup_ref and Path(result.backup_ref).is_file() with closing(sqlite3.connect(path)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_4_1_006",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] assert ( json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ "mode" diff --git a/tests/test_migration_revision_names.py b/tests/test_migration_revision_names.py index 017f748591..d620895262 100644 --- a/tests/test_migration_revision_names.py +++ b/tests/test_migration_revision_names.py @@ -61,9 +61,10 @@ def invoke(action, *options): "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", + "v1_5_0_001", ] assert [item["revision"] for item in plan["pending_migrations"]] == list(reviewed.revisions) - target = {"revision": "v1_4_1_006", "name": "v1_4_1_006", "software_version": "1.4.1", "sequence": 6} + target = {"revision": "v1_5_0_001", "name": "v1_5_0_001", "software_version": "1.5.0", "sequence": 1} assert plan["target_migration"] == target applied = invoke( @@ -78,7 +79,7 @@ def invoke(action, *options): ) assert applied["migration"] == target and applied["server_ready"] with closing(sqlite3.connect(runner.database)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_4_1_006",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] evidence = runner.evidence_path.read_bytes() recorded = json.loads(evidence) assert recorded["source_revision"] == expected_source["revision"] @@ -90,6 +91,7 @@ def invoke(action, *options): "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", + "v1_5_0_001", } assert runner.verify().state == "ready" assert invoke("verify")["migration"] == target diff --git a/tests/test_registered_source_maintenance_cli.py b/tests/test_registered_source_maintenance_cli.py index 68f737b4c0..c593e6c0c4 100644 --- a/tests/test_registered_source_maintenance_cli.py +++ b/tests/test_registered_source_maintenance_cli.py @@ -63,7 +63,7 @@ def test_release140_compatibility_commands_preserve_source_and_share_full_upgrad planned = CliRunner().invoke(cli, [*arguments, "plan", "--backup", "skip"]) assert planned.exit_code == 0, planned.output plan = json.loads(planned.output) - assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_4_1_006" + assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_5_0_001" assert plan["readiness_scope"] == "server" full = CliRunner().invoke(cli, ["server", "db-migrate", "plan", "--backup", "skip"]) assert full.exit_code == 0, full.output