XY-791: [ELF hardening 3/4] Enforce strict config field presence

.github/fixtures/trace_gate/config.toml

@@ -9,9 +9,10 @@ dsn            = "postgres://postgres:postgres@127.0.0.1:5432/elf"
 pool_max_conns = 5
 
 [storage.qdrant]
-collection = "ci_trace_gate"
-url        = "http://127.0.0.1:6334"
-vector_dim = 4
+collection      = "ci_trace_gate"
+docs_collection = "ci_trace_gate_docs"
+url             = "http://127.0.0.1:6334"
+vector_dim      = 4
 
 [providers.embedding]
 api_base        = "http://127.0.0.1"
@@ -68,6 +69,12 @@ max_notes_per_add_event = 3
 top_k                   = 3
 update_sim_threshold    = 0.85
 
+[memory.policy]
+
+[[memory.policy.rules]]
+min_confidence = 0.0
+min_importance = 0.0
+
 [chunking]
 enabled        = true
 max_tokens     = 256
@@ -98,6 +105,18 @@ capture_candidates       = false
 retention_days           = 2
 write_mode               = "outbox"
 
+[search.recursive]
+enabled               = false
+max_children_per_node = 4
+max_depth             = 2
+max_nodes_per_scope   = 32
+max_total_nodes       = 256
+
+[search.graph_context]
+enabled                     = false
+max_evidence_notes_per_fact = 16
+max_facts_per_item          = 16
+
 [ranking]
 recency_tau_days   = 0.0
 tie_breaker_weight = 0.0
@@ -157,6 +176,7 @@ purge_deleted_after_days    = 30
 purge_deprecated_after_days = 180
 
 [security]
+auth_keys                = []
 auth_mode                = "off"
 bind_localhost_only      = true
 evidence_max_quote_chars = 320

apps/elf-api/tests/http.rs

@@ -20,13 +20,13 @@ use uuid::Uuid;
 
 use elf_api::{routes, state::AppState};
 use elf_config::{
-	Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, Postgres,
-	ProviderConfig, Providers, Qdrant, Ranking, RankingBlend, RankingBlendSegment,
+	Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, MemoryPolicy,
+	Postgres, ProviderConfig, Providers, Qdrant, Ranking, RankingBlend, RankingBlendSegment,
 	RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits,
 	RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles,
 	ScopePrecedence, ScopeWriteAllowed, Scopes, Search, SearchCache, SearchDynamic,
-	SearchExpansion, SearchExplain, SearchPrefilter, Security, SecurityAuthKey, SecurityAuthRole,
-	Service, Storage, TtlDays,
+	SearchExpansion, SearchExplain, SearchGraphContext, SearchPrefilter, SearchRecursive, Security,
+	SecurityAuthKey, SecurityAuthRole, Service, Storage, TtlDays,
 };
 use elf_storage::qdrant::{BM25_MODEL, BM25_VECTOR_NAME, DENSE_VECTOR_NAME};
 use elf_testkit::TestDatabase;
@@ -137,31 +137,9 @@ fn test_config(dsn: String, qdrant_url: String, collection: String) -> Config {
 			update_sim_threshold: 0.85,
 			candidate_k: 60,
 			top_k: 12,
-			policy: Default::default(),
-		},
-		search: Search {
-			expansion: SearchExpansion {
-				mode: "off".to_string(),
-				max_queries: 4,
-				include_original: true,
-			},
-			dynamic: SearchDynamic { min_candidates: 10, min_top_score: 0.12 },
-			prefilter: SearchPrefilter { max_candidates: 0 },
-			cache: SearchCache {
-				enabled: true,
-				expansion_ttl_days: 7,
-				rerank_ttl_days: 7,
-				max_payload_bytes: Some(262_144),
-			},
-			explain: SearchExplain {
-				retention_days: 7,
-				capture_candidates: false,
-				candidate_retention_days: 2,
-				write_mode: "outbox".to_string(),
-			},
-			recursive: Default::default(),
-			graph_context: Default::default(),
+			policy: MemoryPolicy { rules: vec![] },
 		},
+		search: test_search(),
 		ranking: test_ranking(),
 		lifecycle: Lifecycle {
 			ttl_days: TtlDays {
@@ -196,6 +174,42 @@ fn test_config(dsn: String, qdrant_url: String, collection: String) -> Config {
 	}
 }
 
+fn test_search() -> Search {
+	Search {
+		expansion: SearchExpansion {
+			mode: "off".to_string(),
+			max_queries: 4,
+			include_original: true,
+		},
+		dynamic: SearchDynamic { min_candidates: 10, min_top_score: 0.12 },
+		prefilter: SearchPrefilter { max_candidates: 0 },
+		cache: SearchCache {
+			enabled: true,
+			expansion_ttl_days: 7,
+			rerank_ttl_days: 7,
+			max_payload_bytes: Some(262_144),
+		},
+		explain: SearchExplain {
+			retention_days: 7,
+			capture_candidates: false,
+			candidate_retention_days: 2,
+			write_mode: "outbox".to_string(),
+		},
+		recursive: SearchRecursive {
+			enabled: false,
+			max_depth: 2,
+			max_children_per_node: 4,
+			max_nodes_per_scope: 32,
+			max_total_nodes: 256,
+		},
+		graph_context: SearchGraphContext {
+			enabled: false,
+			max_facts_per_item: 16,
+			max_evidence_notes_per_fact: 16,
+		},
+	}
+}
+
 fn dummy_embedding_provider() -> EmbeddingProviderConfig {
 	EmbeddingProviderConfig {
 		provider_id: "local".to_string(),
@@ -753,12 +767,18 @@ async fn fetch_admin_search_raw_source_ref(
 		)
 		.await
 		.expect("Failed to call admin search raw.");
-
-	assert_eq!(response.status(), StatusCode::OK);
-
+	let status = response.status();
 	let body = body::to_bytes(response.into_body(), usize::MAX)
 		.await
 		.expect("Failed to read admin search raw response body.");
+
+	assert_eq!(
+		status,
+		StatusCode::OK,
+		"Unexpected admin search raw status with body: {}",
+		String::from_utf8_lossy(&body)
+	);
+
 	let json: serde_json::Value =
 		serde_json::from_slice(&body).expect("Failed to parse admin search raw response.");
 	let item = json["items"]
@@ -1496,7 +1516,8 @@ async fn searches_notes_payload_level_shapes_source_ref_and_structured() {
 		}
 	});
 	let structured_summary = "Compact structured summary used for payload-level l1 and l2 shaping.";
-	let note_text = "A payload shaping note used in contract tests for search details output shaping. It includes deliberate    spacing and\nline breaks so l0 compaction can be observed.";
+	let note_text =
+		"Payload shaping note used in contract tests for search details output shaping.";
 	let note_id =
 		create_note_for_payload_level_tests(&app, &state, note_text, source_ref.clone()).await;
 
@@ -1589,7 +1610,7 @@ async fn searches_notes_payload_level_shapes_source_ref_and_structured() {
 	assert!(notes_l1["structured"].is_object());
 	assert!(notes_l2["structured"].is_object());
 	assert!(notes_l0_text.len() <= 240);
-	assert_ne!(notes_l0_text, note_text);
+	assert_eq!(notes_l0_text, note_text);
 	assert_eq!(notes_l1_text, structured_summary);
 	assert_eq!(notes_l2_text, note_text);
 

docs/spec/system_elf_memory_service_v2.md

@@ -73,7 +73,7 @@ Rules:
 - chunking.enabled must be true.
 - chunking.max_tokens must be greater than zero.
 - chunking.overlap_tokens must be less than chunking.max_tokens.
-- chunking.tokenizer_repo may be empty or omitted to inherit providers.embedding.model.
+- chunking.tokenizer_repo must be present and non-empty.
 
 Template (all values required):
 
@@ -90,6 +90,7 @@ pool_max_conns = <REQUIRED_INT>
 [storage.qdrant]
 url = "<REQUIRED_URL>"
 collection = "mem_notes_v2"
+docs_collection = "doc_chunks_v1"
 vector_dim = <REQUIRED_INT>
 
 [providers.embedding]
@@ -152,12 +153,19 @@ update_sim_threshold = 0.85
 candidate_k = 60
 top_k = 12
 
+[memory.policy]
+
+[[memory.policy.rules]]
+note_type = "fact|plan|preference|constraint|decision|profile"
+scope = "agent_private|project_shared|org_shared"
+min_confidence = <OPTIONAL_FLOAT>
+min_importance = <OPTIONAL_FLOAT>
+
 [chunking]
 enabled = true
 max_tokens = <REQUIRED_INT>
 overlap_tokens = <REQUIRED_INT>
-# Optional. Empty or omitted uses providers.embedding.model.
-tokenizer_repo = "<OPTIONAL_STRING>"
+tokenizer_repo = "<REQUIRED_NON_EMPTY_STRING>"
 
 [search.expansion]
 mode = "off|always|dynamic"
@@ -180,14 +188,68 @@ max_payload_bytes = <OPTIONAL_INT>
 
 [search.explain]
 retention_days = <REQUIRED_INT>
-capture_candidates = <OPTIONAL_BOOL>
-candidate_retention_days = <OPTIONAL_INT>
-write_mode = <OPTIONAL_STRING>
+capture_candidates = <REQUIRED_BOOL>
+candidate_retention_days = <REQUIRED_INT>
+write_mode = "outbox|inline"
+
+[search.recursive]
+enabled = <REQUIRED_BOOL>
+max_depth = <REQUIRED_INT>
+max_children_per_node = <REQUIRED_INT>
+max_nodes_per_scope = <REQUIRED_INT>
+max_total_nodes = <REQUIRED_INT>
+
+[search.graph_context]
+enabled = <REQUIRED_BOOL>
+max_facts_per_item = <REQUIRED_INT>
+max_evidence_notes_per_fact = <REQUIRED_INT>
 
 [ranking]
 recency_tau_days = 60
 tie_breaker_weight = 0.1
 
+[ranking.deterministic]
+enabled = <REQUIRED_BOOL>
+
+[ranking.deterministic.lexical]
+enabled = <REQUIRED_BOOL>
+weight = <REQUIRED_FLOAT>
+min_ratio = <REQUIRED_FLOAT>
+max_query_terms = <REQUIRED_INT>
+max_text_terms = <REQUIRED_INT>
+
+[ranking.deterministic.hits]
+enabled = <REQUIRED_BOOL>
+weight = <REQUIRED_FLOAT>
+half_saturation = <REQUIRED_FLOAT>
+last_hit_tau_days = <REQUIRED_FLOAT>
+
+[ranking.deterministic.decay]
+enabled = <REQUIRED_BOOL>
+weight = <REQUIRED_FLOAT>
+tau_days = <REQUIRED_FLOAT>
+
+[ranking.blend]
+enabled = <REQUIRED_BOOL>
+rerank_normalization = "<REQUIRED_STRING>"
+retrieval_normalization = "<REQUIRED_STRING>"
+
+[[ranking.blend.segments]]
+max_retrieval_rank = <REQUIRED_INT>
+retrieval_weight = <REQUIRED_FLOAT>
+
+[ranking.diversity]
+enabled = <REQUIRED_BOOL>
+sim_threshold = <REQUIRED_FLOAT>
+mmr_lambda = <REQUIRED_FLOAT>
+max_skips = <REQUIRED_INT>
+
+[ranking.retrieval_sources]
+fusion_weight = <REQUIRED_FLOAT>
+structured_field_weight = <REQUIRED_FLOAT>
+fusion_priority = <REQUIRED_INT>
+structured_field_priority = <REQUIRED_INT>
+
 [lifecycle.ttl_days]
 plan = 14
 fact = 180
@@ -208,6 +270,19 @@ redact_secrets_on_write = true
 evidence_min_quotes = 1
 evidence_max_quotes = 2
 evidence_max_quote_chars = 320
+auth_mode = "off|static_keys"
+# Must exist. Empty array is allowed only when auth_mode = "off".
+auth_keys = []
+
+# Required when auth_mode = "static_keys"; replace auth_keys = [] with one or more entries.
+# [[security.auth_keys]]
+# token_id = "<REQUIRED_ID>"
+# token = "<REQUIRED_NON_EMPTY>"
+# tenant_id = "<REQUIRED_ID>"
+# project_id = "<REQUIRED_ID>"
+# agent_id = "<REQUIRED_ID>"
+# read_profile = "private_only|private_plus_project|all_scopes"
+# role = "user|admin|super_admin"
 
 [context]
 # Optional. Context metadata used to disambiguate retrieval across projects and scopes.
@@ -228,7 +303,6 @@ scope_boost_weight = <OPTIONAL_FLOAT>
 tenant_id = "<REQUIRED_ID>"
 project_id = "<REQUIRED_ID>"
 agent_id = "<REQUIRED_ID>"
-# Optional. Default is private_plus_project.
 read_profile = "private_only|private_plus_project|all_scopes"
 
 ============================================================