feat(retention): retire content weeks on one creation-anchored clock (DW-992)

.github/fixtures/fusillade-migration-sha384.txt

@@ -226,3 +226,9 @@ e7e9fb0732e4f0755fe452c6463391da45615f4800c737061946deaadd11199bdeeb53ca3705a46d
 936f98aa92536a6d99439d8a96d0f07e18c8962feb4901602ef1b33c9e5a0ccd2d8dd289ae6cebc80b1d1c596966b3f9  fusillade-arsenal/migrations/20261013100020_reindex_batches_finished_late_index.up.sql
 d34c3ccb5ca7b28858866fd16115615853c49736d0cbe842fa9e67b651aeaddf98e3906d471e728239c7ec5e596ad622  fusillade-arsenal/migrations/20261013100030_validate_batches_finished_late_index.down.sql
 e8526863e7214449572b9fa4a4cec20f08ea8ebf5ba71e905f8d5845f4b2387b864e96be6ed2a8b993eacb182a891436  fusillade-arsenal/migrations/20261013100030_validate_batches_finished_late_index.up.sql
+b4b17aeda491ce00f00cc1d69d885f09522bb6bd315dee7ad4b2ef4ea2775ce6348b2b1864cf9b7686f75f299d3d84a2  fusillade-arsenal/migrations/20261023100010_add_batches_unfinished_created_at_index.down.sql
+b6f9ecc0b4af2ceb0dd45437e9faf19c2162e670120fc25551b575b2ffd766e0c4b34a0f90ba064ebf9a8da7892b556f  fusillade-arsenal/migrations/20261023100010_add_batches_unfinished_created_at_index.up.sql
+922156d0d67b2c8e53e9277713e2f1f58a59cae4e0b953b9a6bf10bf5804857eca5628b7d1064916775d089b7bc04f1a  fusillade-arsenal/migrations/20261023100020_reindex_batches_unfinished_created_at_index.down.sql
+412a0cd1a7174692745f5713e63fbd3fbed2c82e4923a78028965f7c47e1a1327ccac0a258a55f56246220ac2a86e8b1  fusillade-arsenal/migrations/20261023100020_reindex_batches_unfinished_created_at_index.up.sql
+45c71e555f9cb92c2113014e3a28e793ffd39d9ad08d610a178b9d9608992237d127e3386aa72f8462f44b7a0b9041c1  fusillade-arsenal/migrations/20261023100030_validate_batches_unfinished_created_at_index.down.sql
+90e1ebe2c2f229dc48aeb30c3c17d352ee856ec6ccab20675479d52b8e638f90461852ebcd3b6f456b54006ff83591d1  fusillade-arsenal/migrations/20261023100030_validate_batches_unfinished_created_at_index.up.sql

config.yaml

@@ -923,24 +923,38 @@ background_services:
     # configured until every journaled retirement has completed, even if the
     # enable flag is turned back off.
     # retained_response_retirement_enabled: false
-    # Weekly batch-archive retirement irreversibly drops whole expired
-    # archive partitions once every batch in the week is past the
-    # finalization-anchored retention period below. Batch metadata rows are
-    # stamped retention-expired, never deleted. Requires the same
-    # session-capable maintenance endpoint as response retirement.
+    # Content retention for both weekly families below is R =
+    # batches.files.max_expiry_seconds rounded up to whole days. A week drops
+    # once it is R days past its end, so no file's expires_at can outrun its
+    # content. While either family is enabled, a batch can no longer be
+    # retried once it, or its input file, is R - 1 days old (a retry would
+    # hold its weeks open past the period); the API answers 400.
+    #
+    # Weekly batch-archive retirement irreversibly drops a whole archive
+    # partition once the week is past R, measured from the week itself, and
+    # no batch created in that week is still running. The oldest completion
+    # in a dropped week is 7 + R days old, the newest R days. Batch metadata
+    # rows are never touched. Requires the same session-capable maintenance
+    # endpoint as response retirement.
     # batch_archive_retirement_enabled: false
-    # batch_archive_retention_days: <positive days>
+    # DEPRECATED and ignored: batch_archive_retention_days and
+    # template_retention_days no longer set retention, which follows
+    # batches.files.max_expiry_seconds. The keys still parse so existing
+    # values files keep loading, but log a warning at startup. Remove them.
+    # batch_archive_retention_days: <ignored>
     # DEPRECATED and ignored: request templates are always written to the
     # weekly generation-2 partitions now that the generation-1 heap has been
     # retired. The key still parses so existing values files keep loading,
     # but it logs a warning at startup. Remove it.
     # template_generation_writes_enabled: false
-    # File-content expiry + weekly template partition retirement. Tombstones
-    # aged input-file metadata (rows survive) once every referencing batch's
-    # content is expired, then drops whole template weeks that no live file
-    # references. Requires the same session-capable maintenance endpoint.
+    # Weekly template retirement irreversibly drops a whole week of request
+    # templates (batch input content) once the week is past R, measured from
+    # the week itself, and no batch still running reads input uploaded in
+    # that week. File metadata rows are never touched; an input file reports
+    # expired from its own expires_at. Requires the same session-capable
+    # maintenance endpoint.
     # template_retirement_enabled: false
-    # template_retention_days: <positive days>
+    # template_retention_days: <ignored, deprecated; see above>
     # Schema mode reuses the main direct credentials automatically. Override for
     # dedicated databases or a different Fusillade query role. Use that role on
     # the DIRECT host; startup attests database, role, schema, and ownership.

docs/src/reference/configuration.md

@@ -378,6 +378,8 @@ The expiry settings must be positive and ordered `min_expiry_seconds <= default_
 - `GET /ai/v1/batches/{batch_id}/results` returns 410 once the batch's output or error file is past its `expires_at`.
 - Output and error files expire `default_expiry_seconds` after the batch is created. OpenAI counts from when the batch completes instead, so here they expire up to one completion window earlier. Output and error files created before they were given an expiry are treated as expiring `default_expiry_seconds` after their creation.
 
+Stored batch content is deleted in weekly partitions by two retirement switches under `background_services.batch_daemon`, both `false` by default: `template_retirement_enabled` deletes batch inputs (request templates) and `batch_archive_retirement_enabled` deletes archived batch results. An enabled family deletes a week no earlier than `max_expiry_seconds` (rounded up to whole days) after the end of the week the content was created in. While either switch is on, a batch retry is refused with 400 once the batch or its input file is older than that period minus one day. The former `background_services.batch_daemon.batch_archive_retention_days` and `background_services.batch_daemon.template_retention_days` keys are deprecated and ignored; remove them.
+
 ## Background Services
 
 ### Onwards Sync

dwctl/src/api/handlers/batches.rs

@@ -1788,7 +1788,7 @@ pub async fn delete_batch<P: PoolProvider>(
     description = "Retry a batch: failed AND canceled requests are reset to pending and will be processed again; completed work is never redone. Retrying a cancelled batch overturns the cancellation (cancel-then-retry acts as pause/resume), so this also succeeds on a just-cancelled batch whose requests had not yet settled. Use it after fixing transient issues, increasing rate limits, or to resume a cancelled batch.",
     responses(
         (status = 200, description = "Requests queued for retry (or a cancellation was overturned and the batch resumed).", body = BatchResponse),
-        (status = 400, description = "Nothing to retry: no failed or canceled requests, and the batch was not cancelled."),
+        (status = 400, description = "The batch cannot be retried: it has no failed or canceled requests and was not cancelled, it is past the retry age cut-off, or its input or results have been deleted by content retention."),
         (status = 404, description = "Batch not found or you don't have access to it."),
         (status = 500, description = "An unexpected error occurred. Retry the request or contact support if the issue persists.")
     ),
@@ -1830,8 +1830,13 @@ pub async fn retry_failed_batch_requests<P: PoolProvider>(
         .request_manager
         .retry_failed_requests_for_batch(fusillade::BatchId(batch_id))
         .await
-        .map_err(|e| Error::Internal {
-            operation: format!("retry failed requests: {}", e),
+        .map_err(|e| match e {
+            // Refusals the client can act on: the batch is past its retry
+            // age cut-off, or its input or results have been retired.
+            fusillade::FusilladeError::ValidationError(message) => Error::BadRequest { message },
+            e => Error::Internal {
+                operation: format!("retry failed requests: {}", e),
+            },
         })?;
 
     // retried_count == 0 is still SUCCESS when the retry overturned a
@@ -1880,7 +1885,7 @@ Use this for fine-grained control over which requests to retry, rather than retr
     request_body = RetryRequestsRequest,
     responses(
         (status = 200, description = "Specified requests queued for retry.", body = BatchResponse),
-        (status = 400, description = "No valid request IDs provided."),
+        (status = 400, description = "No valid request IDs provided, or the batch is past the retry age cut-off. Requests whose input has been deleted by content retention are left as they are; the batch is returned with status 200."),
         (status = 404, description = "Batch not found or you don't have access to it."),
         (status = 500, description = "An unexpected error occurred. Retry the request or contact support if the issue persists.")
     ),
@@ -1942,8 +1947,13 @@ pub async fn retry_specific_requests<P: PoolProvider>(
         .request_manager
         .retry_failed_requests(fusillade::BatchId(batch_id), request_ids.clone())
         .await
-        .map_err(|e| Error::Internal {
-            operation: format!("retry failed requests: {}", e),
+        .map_err(|e| match e {
+            // The batch is past its retry age cut-off. A request whose input
+            // has been retired is refused on its own, in `results`.
+            fusillade::FusilladeError::ValidationError(message) => Error::BadRequest { message },
+            e => Error::Internal {
+                operation: format!("retry failed requests: {}", e),
+            },
         })?;
 
     // Check for any failures

dwctl/src/config.rs

@@ -1275,7 +1275,8 @@ pub struct FilesConfig {
     /// Smallest `expires_after[seconds]` an upload may request (default: 1 hour)
     pub min_expiry_seconds: i64,
     /// Largest `expires_after[seconds]` an upload may request (default: 30
-    /// days). Must not exceed what content retention honours.
+    /// days). Content retention is this rounded up to whole days, so no
+    /// `expires_at` outlives the content.
     pub max_expiry_seconds: i64,
     /// Buffer size for file upload streams (default: 100)
     pub upload_buffer_size: usize,
@@ -2338,8 +2339,9 @@ pub struct DaemonConfig {
     #[serde(default)]
     pub retained_response_retirement_enabled: bool,
 
-    /// Allow weekly batch-archive partition retirement. Disabled by default
-    /// and additionally requires an explicit retention period.
+    /// Allow weekly batch-archive partition retirement. Disabled by default.
+    /// Retention follows `batches.files.max_expiry_seconds`; see
+    /// [`content_retention_days`].
     #[serde(default)]
     pub batch_archive_retirement_enabled: bool,
 
@@ -2350,20 +2352,21 @@ pub struct DaemonConfig {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub template_generation_writes_enabled: Option<bool>,
 
-    /// Allow file-content expiry and weekly template partition retirement.
-    /// Disabled by default and additionally requires an explicit retention
-    /// period.
+    /// Allow weekly template partition retirement. Disabled by default.
+    /// Retention follows `batches.files.max_expiry_seconds`; see
+    /// [`content_retention_days`].
     #[serde(default)]
     pub template_retirement_enabled: bool,
 
-    /// Creation-anchored input-content retention period in days. No default:
-    /// enabling template retirement without it fails startup.
-    #[serde(default)]
+    /// Deprecated and ignored: content retention follows
+    /// `batches.files.max_expiry_seconds`. The key is still accepted so
+    /// existing values files keep parsing; a warning is logged at startup
+    /// when it is set. Remove it from your configuration.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub template_retention_days: Option<u32>,
 
-    /// Finalization-anchored batch content retention period in days. No
-    /// default: enabling batch-archive retirement without it fails startup.
-    #[serde(default)]
+    /// Deprecated and ignored; see `template_retention_days`.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub batch_archive_retention_days: Option<u32>,
 
     /// Explicit direct/session-capable primary endpoint used only for
@@ -2754,8 +2757,24 @@ impl Default for DaemonConfig {
     }
 }
 
+/// Content retention in whole days, derived from the longest file expiry an
+/// upload may ask for, rounded up: `ceil(max_expiry_seconds / 1 day)`.
+///
+/// Both weekly content families (batch archive and request templates) drop a
+/// week once `week_start + 7 days + this` has passed. Every file's
+/// `expires_at` is at most `created_at + max_expiry_seconds`, which is at
+/// most the week's end plus this, so the date the API shows can never
+/// outrun the content.
+pub fn content_retention_days(max_expiry_seconds: i64) -> u32 {
+    const SECONDS_PER_DAY: i64 = 24 * 60 * 60;
+    let days = max_expiry_seconds.max(1).saturating_add(SECONDS_PER_DAY - 1) / SECONDS_PER_DAY;
+    u32::try_from(days.min(i64::from(i32::MAX))).unwrap_or(i32::MAX as u32)
+}
+
 impl DaemonConfig {
-    pub fn to_fusillade_retention_maintenance_config(&self) -> fusillade::RetentionMaintenanceConfig {
+    /// Build the daemon's retention maintenance controls. Content retention
+    /// is derived from `max_expiry_seconds`; see [`content_retention_days`].
+    pub fn to_fusillade_retention_maintenance_config(&self, max_expiry_seconds: i64) -> fusillade::RetentionMaintenanceConfig {
         fusillade::RetentionMaintenanceConfig::new(self.retention.clone())
             .with_batchless_archive_sweep_enabled(self.batchless_archive_sweep_enabled)
             .with_batchless_archive_backfill_enabled(self.batchless_archive_backfill_enabled)
@@ -2764,9 +2783,8 @@ impl DaemonConfig {
             .with_retained_response_partitions_days_ahead(self.retained_response_partitions_days_ahead)
             .with_retained_response_retirement_enabled(self.retained_response_retirement_enabled)
             .with_batch_archive_retirement_enabled(self.batch_archive_retirement_enabled)
-            .with_batch_archive_retention_days(self.batch_archive_retention_days)
             .with_template_retirement_enabled(self.template_retirement_enabled)
-            .with_template_retention_days(self.template_retention_days)
+            .with_content_retention_days(Some(content_retention_days(max_expiry_seconds)))
     }
 
     /// Convert to fusillade daemon config
@@ -3862,22 +3880,6 @@ impl Config {
                 operation: "Config validation: partition retirement on a dedicated database requires an explicit direct session endpoint using the same role as fusillade query traffic".to_string(),
             });
         }
-        if owns_archive_maintenance
-            && daemon.batch_archive_retirement_enabled
-            && daemon.batch_archive_retention_days.is_none_or(|days| days < 1)
-        {
-            return Err(Error::Internal {
-                operation:
-                    "Config validation: batch-archive partition retirement requires an explicit positive batch_archive_retention_days"
-                        .to_string(),
-            });
-        }
-        if owns_archive_maintenance && daemon.template_retirement_enabled && daemon.template_retention_days.is_none_or(|days| days < 1) {
-            return Err(Error::Internal {
-                operation: "Config validation: template partition retirement requires an explicit positive template_retention_days"
-                    .to_string(),
-            });
-        }
         if owns_archive_maintenance
             && batchless_movement_enabled
             && (daemon.batchless_archive_groups_per_tick <= 0 || daemon.batchless_archive_bytes_per_tick <= 0)
@@ -4403,7 +4405,7 @@ mod tests {
         assert_eq!(daemon.batchless_archive_backfill_concurrency, 1);
         assert!(daemon.retained_response_partitions_days_ahead > 0);
 
-        let mapped = daemon.to_fusillade_retention_maintenance_config();
+        let mapped = daemon.to_fusillade_retention_maintenance_config(30 * 24 * 60 * 60 + 1);
         assert_eq!(mapped.policy(), &daemon.retention);
         assert_eq!(mapped.batchless_archive_sweep_enabled(), daemon.batchless_archive_sweep_enabled);
         assert_eq!(
@@ -4425,7 +4427,12 @@ mod tests {
             daemon.retained_response_retirement_enabled
         );
         assert_eq!(mapped.batch_archive_retirement_enabled(), daemon.batch_archive_retirement_enabled);
-        assert_eq!(mapped.batch_archive_retention_days(), daemon.batch_archive_retention_days);
+        assert_eq!(mapped.template_retirement_enabled(), daemon.template_retirement_enabled);
+        assert_eq!(
+            mapped.content_retention_days(),
+            Some(31),
+            "retention rounds max expiry up to whole days"
+        );
     }
 
     #[test]
@@ -5159,6 +5166,16 @@ secret_key: "test-secret-key"
         assert!(result.is_err());
     }
 
+    #[test]
+    fn content_retention_rounds_max_expiry_up_to_whole_days() {
+        let day = 24 * 60 * 60;
+        assert_eq!(content_retention_days(1), 1);
+        assert_eq!(content_retention_days(day), 1);
+        assert_eq!(content_retention_days(day + 1), 2);
+        assert_eq!(content_retention_days(30 * day), 30);
+        assert_eq!(content_retention_days(i64::MAX), i32::MAX as u32);
+    }
+
     #[test]
     fn retention_config_is_accepted_on_an_instance_that_never_runs_the_daemon() {
         // Split topology: API pods share the daemon pods' environment but run
@@ -5174,10 +5191,12 @@ secret_key: "test-secret-key"
         daemon.batchless_archive_sweep_enabled = true;
         daemon.retained_response_retirement_enabled = true;
         daemon.batch_archive_retirement_enabled = true;
+        // Deprecated no-op key: still parses, never rejects.
         daemon.batch_archive_retention_days = Some(1);
         // Deprecated no-op key: still parses, never rejects.
         daemon.template_generation_writes_enabled = Some(true);
         daemon.template_retirement_enabled = true;
+        // Deprecated no-op key: still parses, never rejects.
         daemon.template_retention_days = Some(1);
         assert!(config.validate().is_ok(), "{:?}", config.validate().err());
     }