Compact Responses history and completed tool steps on the server

.agents/skills/validate-opensecret/scripts/disposable_db_tests.sh

@@ -95,8 +95,8 @@ cleanup() {
   rm -rf -- "$workdir"
 
   if [ "$status" -eq 0 ] && [ "$tests_passed" -eq 1 ]; then
-    printf 'Disposable-DB evidence: %s AEAD/database tests, %s OAuth database tests, and %s callback settings/transport tests passed; no tests skipped; temporary cluster removed.\n' \
-      "$aead_count" "$oauth_count" "$callback_count"
+    printf 'Disposable-DB evidence: %s AEAD/database tests, %s OAuth database tests, %s callback settings/transport tests, and %s Responses compaction persistence tests passed; no tests skipped; temporary cluster removed.\n' \
+      "$aead_count" "$oauth_count" "$callback_count" "$compaction_count"
   fi
   exit "$status"
 }
@@ -182,6 +182,13 @@ oauth_count="$(awk '/^web::oauth_routes::tests::db_.*: test$/ { count++ }
   END { print count + 0 }' "$workdir/oauth-tests.list")"
 test "$oauth_count" -gt 0
 
+cargo test --locked --all-features models::response_compaction::tests::db_ \
+  -- --ignored --list | tee "$workdir/compaction-tests.list"
+compaction_count="$(awk '/^models::response_compaction::tests::db_.*: test$/ { count++ }
+  END { print count + 0 }' "$workdir/compaction-tests.list")"
+# Keep the selected persistence contract explicit: a missing ignored test is a failure.
+test "$compaction_count" -eq 20
+
 cargo test --locked --all-features aead_db_tamper_tests \
   -- --ignored --test-threads=1 --nocapture 2>&1 | tee "$workdir/aead-tests.log"
 if grep -qi 'skipping:' "$workdir/aead-tests.log"; then
@@ -200,6 +207,15 @@ fi
 grep -Eq "test result: ok\\. ${oauth_count} passed; 0 failed; 0 ignored;" \
   "$workdir/oauth-tests.log"
 
+cargo test --locked --all-features models::response_compaction::tests::db_ \
+  -- --ignored --test-threads=1 --nocapture 2>&1 | tee "$workdir/compaction-tests.log"
+if grep -qi 'skipping:' "$workdir/compaction-tests.log"; then
+  printf 'Responses compaction persistence test output contained a skip marker\n' >&2
+  exit 1
+fi
+grep -Eq "test result: ok\\. ${compaction_count} passed; 0 failed; 0 ignored;" \
+  "$workdir/compaction-tests.log"
+
 callback_count=0
 for callback_filter in \
   db::tests::db_oauth_settings_ \

services/opensecret/migrations/2026-10-20-180000_responses_compaction_checkpoints/down.sql

@@ -0,0 +1,11 @@
+-- Derived versioned checkpoints can be discarded; canonical transcript rows are untouched.
+DELETE FROM conversation_summaries WHERE format_version = 1;
+DROP INDEX conversation_summaries_checkpoint_response;
+DROP INDEX conversation_summaries_checkpoint_generation;
+ALTER TABLE conversation_summaries
+    DROP CONSTRAINT valid_compaction_checkpoint,
+    DROP COLUMN after_item_uuid,
+    DROP COLUMN origin_response_uuid,
+    DROP COLUMN source_fingerprint,
+    DROP COLUMN generation,
+    DROP COLUMN format_version;

services/opensecret/migrations/2026-10-20-180000_responses_compaction_checkpoints/up.sql

@@ -0,0 +1,21 @@
+-- Versioned user-key ciphertext replaces the unused legacy summary projection.
+-- Coverage identities remain inside the encrypted envelope; only its digest is indexed.
+ALTER TABLE conversation_summaries
+    ADD COLUMN format_version INTEGER NOT NULL DEFAULT 0,
+    ADD COLUMN generation BIGINT NOT NULL DEFAULT 0,
+    ADD COLUMN source_fingerprint BYTEA,
+    -- The first checkpoint can precede response persistence; provenance is not a FK.
+    ADD COLUMN origin_response_uuid UUID,
+    ADD COLUMN after_item_uuid UUID,
+    ADD CONSTRAINT valid_compaction_checkpoint CHECK (
+        format_version = 0 OR (
+            format_version = 1 AND generation > 0 AND content_tokens >= 0
+            AND message_count > 0 AND source_fingerprint IS NOT NULL
+            AND octet_length(source_fingerprint) = 32 AND origin_response_uuid IS NOT NULL
+        )
+    );
+CREATE UNIQUE INDEX conversation_summaries_checkpoint_generation
+    ON conversation_summaries(conversation_id, generation) WHERE format_version = 1;
+CREATE INDEX conversation_summaries_checkpoint_response
+    ON conversation_summaries(user_id, conversation_id, origin_response_uuid, generation)
+    WHERE format_version = 1;

services/opensecret/src/aead_db_tamper_tests.rs

@@ -1,5 +1,5 @@
 use crate::{
-    db::{setup_db, DBError},
+    db::{setup_test_db, DBError},
     encrypt::{decrypt_with_key, encrypt_with_key},
     generate_reset_hash,
     login_routes::RegisterCredentials,
@@ -2111,7 +2111,7 @@ async fn insert_conversation_item_of_each_type(
 }
 
 async fn build_local_test_app_state(database_url: String) -> AppState {
-    let db = setup_db(database_url);
+    let db = setup_test_db(database_url);
     AppStateBuilder::default()
         .app_mode(AppMode::Local)
         .db(db)

services/opensecret/src/db.rs

@@ -31,6 +31,10 @@ use crate::models::project_settings::OAuthSettings;
 use crate::models::project_settings::{
     EmailSettings, NewProjectSetting, ProjectSetting, ProjectSettingError, SettingCategory,
 };
+use crate::models::response_compaction::{
+    CompactionCheckpoint, CompactionError, CompactionPublishResult, CompactionSnapshot,
+    ContextItemKey, NewCompactionCheckpoint,
+};
 use crate::models::responses::{
     validate_conversation_project_limit, AssistantMessage, Conversation, ConversationProject,
     ConversationProjectFilter, NewAssistantMessage, NewConversation, NewConversationProject,
@@ -137,6 +141,8 @@ pub enum DBError {
     ProjectSettingNotFound,
     #[error("Responses API error: {0}")]
     ResponsesError(#[from] crate::models::responses::ResponsesError),
+    #[error("Responses compaction error: {0}")]
+    CompactionError(#[from] CompactionError),
     #[error("User seed wrapping error: {0}")]
     UserSeedWrappingError(#[from] UserSeedWrappingError),
     #[error("App data migration error: {0}")]
@@ -746,6 +752,29 @@ pub trait DBConnection {
     fn create_tool_output(&self, new_output: NewToolOutput) -> Result<ToolOutput, DBError>;
     fn get_tool_output_by_uuid(&self, uuid: Uuid, user_id: Uuid) -> Result<ToolOutput, DBError>;
 
+    // Complete user-scoped checkpoint snapshots; ciphertext is fetched only for selected items.
+    fn load_compaction_snapshot(
+        &self,
+        conversation_id: i64,
+        user_id: Uuid,
+    ) -> Result<CompactionSnapshot, DBError>;
+    fn load_compaction_items(
+        &self,
+        conversation_id: i64,
+        user_id: Uuid,
+        keys: &[ContextItemKey],
+    ) -> Result<Vec<RawThreadMessage>, DBError>;
+    fn list_response_compactions(
+        &self,
+        conversation_id: i64,
+        user_id: Uuid,
+        response_uuid: Uuid,
+    ) -> Result<Vec<CompactionCheckpoint>, DBError>;
+    fn publish_compaction_checkpoint(
+        &self,
+        checkpoint: NewCompactionCheckpoint,
+    ) -> Result<CompactionPublishResult, DBError>;
+
     // Context reconstruction
     fn get_conversation_context_messages(
         &self,
@@ -2572,7 +2601,10 @@ impl DBConnection for PostgresConnection {
 
     fn delete_response(&self, uuid: Uuid, user_id: Uuid) -> Result<(), DBError> {
         let conn = &mut self.db.get().map_err(|_| DBError::ConnectionError)?;
-        Response::delete_by_uuid_and_user(conn, uuid, user_id).map_err(DBError::from)
+        CompactionCheckpoint::delete_response(conn, uuid, user_id).map_err(|error| match error {
+            CompactionError::Responses(error) => DBError::ResponsesError(error),
+            error => DBError::from(error),
+        })
     }
 
     fn create_conversation_with_response_and_message(
@@ -3052,6 +3084,44 @@ impl DBConnection for PostgresConnection {
         RawThreadMessage::get_response_context(conn, response_id).map_err(DBError::from)
     }
 
+    fn load_compaction_snapshot(
+        &self,
+        conversation_id: i64,
+        user_id: Uuid,
+    ) -> Result<CompactionSnapshot, DBError> {
+        let conn = &mut self.db.get().map_err(|_| DBError::ConnectionError)?;
+        CompactionSnapshot::load(conn, conversation_id, user_id).map_err(DBError::from)
+    }
+
+    fn load_compaction_items(
+        &self,
+        conversation_id: i64,
+        user_id: Uuid,
+        keys: &[ContextItemKey],
+    ) -> Result<Vec<RawThreadMessage>, DBError> {
+        let conn = &mut self.db.get().map_err(|_| DBError::ConnectionError)?;
+        CompactionSnapshot::load_items(conn, conversation_id, user_id, keys).map_err(DBError::from)
+    }
+
+    fn list_response_compactions(
+        &self,
+        conversation_id: i64,
+        user_id: Uuid,
+        response_uuid: Uuid,
+    ) -> Result<Vec<CompactionCheckpoint>, DBError> {
+        let conn = &mut self.db.get().map_err(|_| DBError::ConnectionError)?;
+        CompactionCheckpoint::list_for_response(conn, conversation_id, user_id, response_uuid)
+            .map_err(DBError::from)
+    }
+
+    fn publish_compaction_checkpoint(
+        &self,
+        checkpoint: NewCompactionCheckpoint,
+    ) -> Result<CompactionPublishResult, DBError> {
+        let conn = &mut self.db.get().map_err(|_| DBError::ConnectionError)?;
+        CompactionCheckpoint::publish(conn, checkpoint).map_err(DBError::from)
+    }
+
     // Optimized context reconstruction (metadata-based)
     fn get_conversation_context_metadata(
         &self,
@@ -3074,9 +3144,10 @@ impl DBConnection for PostgresConnection {
 
     fn delete_user_message(&self, id: Uuid, user_id: Uuid) -> Result<(), DBError> {
         let conn = &mut self.db.get().map_err(|_| DBError::ConnectionError)?;
-        // First get the message by UUID to find its ID
-        let msg = UserMessage::get_by_uuid_and_user(conn, id, user_id)?;
-        UserMessage::delete_by_id_and_user(conn, msg.id, user_id).map_err(DBError::from)
+        CompactionCheckpoint::delete_user_message(conn, id, user_id).map_err(|error| match error {
+            CompactionError::Responses(error) => DBError::ResponsesError(error),
+            error => DBError::from(error),
+        })
     }
 
     // Maintenance
@@ -3100,6 +3171,23 @@ pub(crate) fn setup_db(url: String) -> Arc<dyn DBConnection + Send + Sync> {
     Arc::new(PostgresConnection { db: pool })
 }
 
+/// Disposable test processes must not exit while idle-refill workers are still
+/// opening native libpq/OpenSSL connections. Production keeps its warm pool.
+#[cfg(test)]
+pub(crate) fn setup_test_db(url: String) -> Arc<dyn DBConnection + Send + Sync> {
+    let manager = ConnectionManager::<PgConnection>::new(url);
+    let pool = Pool::builder()
+        .max_size(20)
+        .min_idle(Some(0))
+        .connection_timeout(std::time::Duration::from_secs(30))
+        .idle_timeout(Some(std::time::Duration::from_secs(600)))
+        .max_lifetime(Some(std::time::Duration::from_secs(1800)))
+        .test_on_check_out(true)
+        .build(manager)
+        .expect("Unable to build disposable test DB connection pool");
+    Arc::new(PostgresConnection { db: pool })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

services/opensecret/src/models/mod.rs

@@ -15,6 +15,7 @@ pub mod platform_password_reset;
 pub mod platform_users;
 pub mod project_settings;
 pub mod remote_enrollment;
+pub mod response_compaction;
 pub mod responses;
 pub mod schema;
 pub mod token_usage;