diff --git a/.github/fixtures/trace_gate/config.toml b/.github/fixtures/trace_gate/config.toml index 83591dec..9768c018 100644 --- a/.github/fixtures/trace_gate/config.toml +++ b/.github/fixtures/trace_gate/config.toml @@ -9,9 +9,10 @@ dsn = "postgres://postgres:postgres@127.0.0.1:5432/elf" pool_max_conns = 5 [storage.qdrant] -collection = "ci_trace_gate" -url = "http://127.0.0.1:6334" -vector_dim = 4 +collection = "ci_trace_gate" +docs_collection = "ci_trace_gate_docs" +url = "http://127.0.0.1:6334" +vector_dim = 4 [providers.embedding] api_base = "http://127.0.0.1" @@ -68,6 +69,12 @@ max_notes_per_add_event = 3 top_k = 3 update_sim_threshold = 0.85 +[memory.policy] + +[[memory.policy.rules]] +min_confidence = 0.0 +min_importance = 0.0 + [chunking] enabled = true max_tokens = 256 @@ -98,6 +105,18 @@ capture_candidates = false retention_days = 2 write_mode = "outbox" +[search.recursive] +enabled = false +max_children_per_node = 4 +max_depth = 2 +max_nodes_per_scope = 32 +max_total_nodes = 256 + +[search.graph_context] +enabled = false +max_evidence_notes_per_fact = 16 +max_facts_per_item = 16 + [ranking] recency_tau_days = 0.0 tie_breaker_weight = 0.0 @@ -157,6 +176,7 @@ purge_deleted_after_days = 30 purge_deprecated_after_days = 180 [security] +auth_keys = [] auth_mode = "off" bind_localhost_only = true evidence_max_quote_chars = 320 diff --git a/apps/elf-api/tests/http.rs b/apps/elf-api/tests/http.rs index fc0d307f..c7f5db50 100644 --- a/apps/elf-api/tests/http.rs +++ b/apps/elf-api/tests/http.rs @@ -20,13 +20,13 @@ use uuid::Uuid; use elf_api::{routes, state::AppState}; use elf_config::{ - Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, Postgres, - ProviderConfig, Providers, Qdrant, Ranking, RankingBlend, RankingBlendSegment, + Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, MemoryPolicy, + Postgres, ProviderConfig, Providers, Qdrant, Ranking, RankingBlend, RankingBlendSegment, RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits, RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles, ScopePrecedence, ScopeWriteAllowed, Scopes, Search, SearchCache, SearchDynamic, - SearchExpansion, SearchExplain, SearchPrefilter, Security, SecurityAuthKey, SecurityAuthRole, - Service, Storage, TtlDays, + SearchExpansion, SearchExplain, SearchGraphContext, SearchPrefilter, SearchRecursive, Security, + SecurityAuthKey, SecurityAuthRole, Service, Storage, TtlDays, }; use elf_storage::qdrant::{BM25_MODEL, BM25_VECTOR_NAME, DENSE_VECTOR_NAME}; use elf_testkit::TestDatabase; @@ -137,31 +137,9 @@ fn test_config(dsn: String, qdrant_url: String, collection: String) -> Config { update_sim_threshold: 0.85, candidate_k: 60, top_k: 12, - policy: Default::default(), - }, - search: Search { - expansion: SearchExpansion { - mode: "off".to_string(), - max_queries: 4, - include_original: true, - }, - dynamic: SearchDynamic { min_candidates: 10, min_top_score: 0.12 }, - prefilter: SearchPrefilter { max_candidates: 0 }, - cache: SearchCache { - enabled: true, - expansion_ttl_days: 7, - rerank_ttl_days: 7, - max_payload_bytes: Some(262_144), - }, - explain: SearchExplain { - retention_days: 7, - capture_candidates: false, - candidate_retention_days: 2, - write_mode: "outbox".to_string(), - }, - recursive: Default::default(), - graph_context: Default::default(), + policy: MemoryPolicy { rules: vec![] }, }, + search: test_search(), ranking: test_ranking(), lifecycle: Lifecycle { ttl_days: TtlDays { @@ -196,6 +174,42 @@ fn test_config(dsn: String, qdrant_url: String, collection: String) -> Config { } } +fn test_search() -> Search { + Search { + expansion: SearchExpansion { + mode: "off".to_string(), + max_queries: 4, + include_original: true, + }, + dynamic: SearchDynamic { min_candidates: 10, min_top_score: 0.12 }, + prefilter: SearchPrefilter { max_candidates: 0 }, + cache: SearchCache { + enabled: true, + expansion_ttl_days: 7, + rerank_ttl_days: 7, + max_payload_bytes: Some(262_144), + }, + explain: SearchExplain { + retention_days: 7, + capture_candidates: false, + candidate_retention_days: 2, + write_mode: "outbox".to_string(), + }, + recursive: SearchRecursive { + enabled: false, + max_depth: 2, + max_children_per_node: 4, + max_nodes_per_scope: 32, + max_total_nodes: 256, + }, + graph_context: SearchGraphContext { + enabled: false, + max_facts_per_item: 16, + max_evidence_notes_per_fact: 16, + }, + } +} + fn dummy_embedding_provider() -> EmbeddingProviderConfig { EmbeddingProviderConfig { provider_id: "local".to_string(), @@ -753,12 +767,18 @@ async fn fetch_admin_search_raw_source_ref( ) .await .expect("Failed to call admin search raw."); - - assert_eq!(response.status(), StatusCode::OK); - + let status = response.status(); let body = body::to_bytes(response.into_body(), usize::MAX) .await .expect("Failed to read admin search raw response body."); + + assert_eq!( + status, + StatusCode::OK, + "Unexpected admin search raw status with body: {}", + String::from_utf8_lossy(&body) + ); + let json: serde_json::Value = serde_json::from_slice(&body).expect("Failed to parse admin search raw response."); let item = json["items"] @@ -1496,7 +1516,8 @@ async fn searches_notes_payload_level_shapes_source_ref_and_structured() { } }); let structured_summary = "Compact structured summary used for payload-level l1 and l2 shaping."; - let note_text = "A payload shaping note used in contract tests for search details output shaping. It includes deliberate spacing and\nline breaks so l0 compaction can be observed."; + let note_text = + "Payload shaping note used in contract tests for search details output shaping."; let note_id = create_note_for_payload_level_tests(&app, &state, note_text, source_ref.clone()).await; @@ -1589,7 +1610,7 @@ async fn searches_notes_payload_level_shapes_source_ref_and_structured() { assert!(notes_l1["structured"].is_object()); assert!(notes_l2["structured"].is_object()); assert!(notes_l0_text.len() <= 240); - assert_ne!(notes_l0_text, note_text); + assert_eq!(notes_l0_text, note_text); assert_eq!(notes_l1_text, structured_summary); assert_eq!(notes_l2_text, note_text); diff --git a/docs/spec/system_elf_memory_service_v2.md b/docs/spec/system_elf_memory_service_v2.md index 2baa3dc3..ea4527de 100644 --- a/docs/spec/system_elf_memory_service_v2.md +++ b/docs/spec/system_elf_memory_service_v2.md @@ -73,7 +73,7 @@ Rules: - chunking.enabled must be true. - chunking.max_tokens must be greater than zero. - chunking.overlap_tokens must be less than chunking.max_tokens. -- chunking.tokenizer_repo may be empty or omitted to inherit providers.embedding.model. +- chunking.tokenizer_repo must be present and non-empty. Template (all values required): @@ -90,6 +90,7 @@ pool_max_conns = [storage.qdrant] url = "" collection = "mem_notes_v2" +docs_collection = "doc_chunks_v1" vector_dim = [providers.embedding] @@ -152,12 +153,19 @@ update_sim_threshold = 0.85 candidate_k = 60 top_k = 12 +[memory.policy] + +[[memory.policy.rules]] +note_type = "fact|plan|preference|constraint|decision|profile" +scope = "agent_private|project_shared|org_shared" +min_confidence = +min_importance = + [chunking] enabled = true max_tokens = overlap_tokens = -# Optional. Empty or omitted uses providers.embedding.model. -tokenizer_repo = "" +tokenizer_repo = "" [search.expansion] mode = "off|always|dynamic" @@ -180,14 +188,68 @@ max_payload_bytes = [search.explain] retention_days = -capture_candidates = -candidate_retention_days = -write_mode = +capture_candidates = +candidate_retention_days = +write_mode = "outbox|inline" + +[search.recursive] +enabled = +max_depth = +max_children_per_node = +max_nodes_per_scope = +max_total_nodes = + +[search.graph_context] +enabled = +max_facts_per_item = +max_evidence_notes_per_fact = [ranking] recency_tau_days = 60 tie_breaker_weight = 0.1 +[ranking.deterministic] +enabled = + +[ranking.deterministic.lexical] +enabled = +weight = +min_ratio = +max_query_terms = +max_text_terms = + +[ranking.deterministic.hits] +enabled = +weight = +half_saturation = +last_hit_tau_days = + +[ranking.deterministic.decay] +enabled = +weight = +tau_days = + +[ranking.blend] +enabled = +rerank_normalization = "" +retrieval_normalization = "" + +[[ranking.blend.segments]] +max_retrieval_rank = +retrieval_weight = + +[ranking.diversity] +enabled = +sim_threshold = +mmr_lambda = +max_skips = + +[ranking.retrieval_sources] +fusion_weight = +structured_field_weight = +fusion_priority = +structured_field_priority = + [lifecycle.ttl_days] plan = 14 fact = 180 @@ -208,6 +270,19 @@ redact_secrets_on_write = true evidence_min_quotes = 1 evidence_max_quotes = 2 evidence_max_quote_chars = 320 +auth_mode = "off|static_keys" +# Must exist. Empty array is allowed only when auth_mode = "off". +auth_keys = [] + +# Required when auth_mode = "static_keys"; replace auth_keys = [] with one or more entries. +# [[security.auth_keys]] +# token_id = "" +# token = "" +# tenant_id = "" +# project_id = "" +# agent_id = "" +# read_profile = "private_only|private_plus_project|all_scopes" +# role = "user|admin|super_admin" [context] # Optional. Context metadata used to disambiguate retrieval across projects and scopes. @@ -228,7 +303,6 @@ scope_boost_weight = tenant_id = "" project_id = "" agent_id = "" -# Optional. Default is private_plus_project. read_profile = "private_only|private_plus_project|all_scopes" ============================================================ diff --git a/packages/elf-config/src/types.rs b/packages/elf-config/src/types.rs index 0576a391..ff7144e0 100644 --- a/packages/elf-config/src/types.rs +++ b/packages/elf-config/src/types.rs @@ -96,7 +96,6 @@ pub struct Qdrant { /// Primary notes collection name. pub collection: String, /// Document-chunk collection name. - #[serde(default = "default_docs_collection")] pub docs_collection: String, /// Vector dimension expected by both note and document collections. pub vector_dim: u32, @@ -236,12 +235,11 @@ pub struct Memory { /// Final top-k size for note retrieval. pub top_k: u32, /// Optional downgrade rules applied after base memory decisions. - #[serde(default)] pub policy: MemoryPolicy, } /// Collection of memory-policy downgrade rules. -#[derive(Debug, Default, Deserialize)] +#[derive(Debug, Deserialize)] pub struct MemoryPolicy { /// Ordered policy rules evaluated against note type, scope, and scores. pub rules: Vec, @@ -287,10 +285,8 @@ pub struct Search { /// Explainability retention settings. pub explain: SearchExplain, /// Recursive retrieval traversal settings. - #[serde(default)] pub recursive: SearchRecursive, /// Graph-context enrichment settings. - #[serde(default)] pub graph_context: SearchGraphContext, } @@ -349,7 +345,6 @@ pub struct SearchExplain { /// Recursive retrieval traversal limits. #[derive(Debug, Deserialize)] -#[serde(default)] pub struct SearchRecursive { /// Whether recursive retrieval is enabled. pub enabled: bool, @@ -362,21 +357,9 @@ pub struct SearchRecursive { /// Maximum nodes retained across the whole traversal. pub max_total_nodes: u32, } -impl Default for SearchRecursive { - fn default() -> Self { - Self { - enabled: false, - max_depth: 2, - max_children_per_node: 4, - max_nodes_per_scope: 32, - max_total_nodes: 256, - } - } -} /// Graph-context enrichment limits applied to search responses. #[derive(Debug, Deserialize)] -#[serde(default)] pub struct SearchGraphContext { /// Whether graph-context enrichment is enabled. pub enabled: bool, @@ -385,11 +368,6 @@ pub struct SearchGraphContext { /// Maximum evidence notes attached to one fact. pub max_evidence_notes_per_fact: u32, } -impl Default for SearchGraphContext { - fn default() -> Self { - Self { enabled: false, max_facts_per_item: 16, max_evidence_notes_per_fact: 16 } - } -} /// Ranking settings for retrieval and rerank fusion. #[derive(Debug, Deserialize)] @@ -554,7 +532,6 @@ pub struct Security { /// Authentication mode such as `off` or `static_keys`. pub auth_mode: String, /// Static bearer-token entries used when `auth_mode` is `static_keys`. - #[serde(default)] pub auth_keys: Vec, } @@ -589,7 +566,3 @@ pub enum SecurityAuthRole { /// Super-admin token for global admin operations. SuperAdmin, } - -fn default_docs_collection() -> String { - "doc_chunks_v1".to_string() -} diff --git a/packages/elf-config/tests/config_validation.rs b/packages/elf-config/tests/config_validation.rs index c2b92c42..100a3355 100644 --- a/packages/elf-config/tests/config_validation.rs +++ b/packages/elf-config/tests/config_validation.rs @@ -16,6 +16,8 @@ use toml::Value; use elf_config::{self, Config, Context, Error, MemoryPolicyRule}; const SAMPLE_CONFIG_TEMPLATE_TOML: &str = include_str!("fixtures/sample_config.template.toml"); +const TRACE_GATE_CONFIG_TOML: &str = + include_str!("../../../.github/fixtures/trace_gate/config.toml"); fn sample_toml(reject_non_english: bool) -> String { sample_toml_with_recursive(reject_non_english, false, 2, 4, 32, 256) @@ -101,12 +103,64 @@ fn write_temp_config(payload: String) -> PathBuf { path } +fn remove_required_config_key(payload: &str, path: &[&str]) -> String { + assert!(!path.is_empty(), "Config path must not be empty."); + + let mut value: Value = toml::from_str(payload).expect("Failed to parse test config."); + let mut table = value.as_table_mut().expect("Template config must be a table."); + + for segment in &path[..path.len() - 1] { + table = table + .get_mut(*segment) + .and_then(Value::as_table_mut) + .unwrap_or_else(|| panic!("Template config must include [{}].", segment)); + } + + let field = path[path.len() - 1]; + let removed = table.remove(field); + + assert!(removed.is_some(), "Template config must include {}.", path.join(".")); + + toml::to_string(&value).expect("Failed to render template config.") +} + +fn assert_missing_field_error(result: Result, field: &str) { + let err = result.expect_err("Expected missing required field parse error."); + let message = match err { + Error::ParseConfig { source, .. } => source.to_string(), + err => panic!("Expected parse config error, got {err}"), + }; + + assert!(message.contains(&format!("missing field `{field}`")), "Unexpected error: {message}"); +} + fn base_config() -> Config { let payload = sample_toml(true); toml::from_str(&payload).expect("Failed to parse test config.") } +#[test] +fn required_config_fields_must_be_explicit() { + let cases = [ + (&["storage", "qdrant", "docs_collection"][..], "docs_collection"), + (&["memory", "policy"][..], "policy"), + (&["search", "recursive"][..], "recursive"), + (&["search", "graph_context"][..], "graph_context"), + (&["security", "auth_keys"][..], "auth_keys"), + ]; + + for (path, field) in cases { + let payload = remove_required_config_key(&sample_toml(true), path); + let config_path = write_temp_config(payload); + let result = elf_config::load(&config_path); + + fs::remove_file(&config_path).expect("Failed to remove test config."); + + assert_missing_field_error(result, field); + } +} + #[test] fn reject_non_english_must_be_true() { let payload = sample_toml(false); @@ -418,6 +472,14 @@ fn elf_example_toml_is_valid() { elf_config::load(&path).expect("Expected elf.example.toml to be a valid config."); } +#[test] +fn trace_gate_fixture_toml_is_valid() { + let path = write_temp_config(TRACE_GATE_CONFIG_TOML.to_string()); + + elf_config::load(&path).expect("Expected trace gate fixture config to be valid."); + fs::remove_file(&path).expect("Failed to remove test config."); +} + #[test] fn retrieval_source_weights_must_be_non_negative() { let mut cfg = base_config(); diff --git a/packages/elf-config/tests/fixtures/sample_config.template.toml b/packages/elf-config/tests/fixtures/sample_config.template.toml index ee666519..ec15e713 100644 --- a/packages/elf-config/tests/fixtures/sample_config.template.toml +++ b/packages/elf-config/tests/fixtures/sample_config.template.toml @@ -9,9 +9,10 @@ dsn = "postgres://user:pass@127.0.0.1:5432/elf" pool_max_conns = 5 [storage.qdrant] -collection = "mem_notes_v2" -url = "http://127.0.0.1:6334" -vector_dim = 4_096 +collection = "mem_notes_v2" +docs_collection = "doc_chunks_v1" +url = "http://127.0.0.1:6334" +vector_dim = 4_096 [providers.embedding] api_base = "http://localhost" @@ -113,6 +114,11 @@ max_depth = 2 max_nodes_per_scope = 32 max_total_nodes = 256 +[search.graph_context] +enabled = false +max_evidence_notes_per_fact = 16 +max_facts_per_item = 16 + [ranking] recency_tau_days = 60.0 tie_breaker_weight = 0.1 diff --git a/packages/elf-domain/src/memory_policy.rs b/packages/elf-domain/src/memory_policy.rs index 19df0e64..cafe3aef 100644 --- a/packages/elf-domain/src/memory_policy.rs +++ b/packages/elf-domain/src/memory_policy.rs @@ -130,8 +130,8 @@ mod tests { RankingBlend, RankingBlendSegment, RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits, RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles, ScopePrecedence, ScopeWriteAllowed, Scopes, Search, - SearchCache, SearchDynamic, SearchExpansion, SearchExplain, SearchPrefilter, Security, - Service, Storage, TtlDays, + SearchCache, SearchDynamic, SearchExpansion, SearchExplain, SearchGraphContext, + SearchPrefilter, SearchRecursive, Security, Service, Storage, TtlDays, }; fn test_config(policy: MemoryPolicy) -> Config { @@ -310,8 +310,18 @@ mod tests { candidate_retention_days: 2, write_mode: "outbox".to_string(), }, - recursive: Default::default(), - graph_context: Default::default(), + recursive: SearchRecursive { + enabled: false, + max_depth: 2, + max_children_per_node: 4, + max_nodes_per_scope: 32, + max_total_nodes: 256, + }, + graph_context: SearchGraphContext { + enabled: false, + max_facts_per_item: 16, + max_evidence_notes_per_fact: 16, + }, } } diff --git a/packages/elf-domain/src/writegate.rs b/packages/elf-domain/src/writegate.rs index 2d907abe..3d66dcc4 100644 --- a/packages/elf-domain/src/writegate.rs +++ b/packages/elf-domain/src/writegate.rs @@ -303,8 +303,8 @@ mod tests { RankingBlendSegment, RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits, RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles, ScopePrecedence, ScopeWriteAllowed, Scopes, Search, - SearchCache, SearchDynamic, SearchExpansion, SearchExplain, SearchPrefilter, Security, - Service, Storage, TtlDays, + SearchCache, SearchDynamic, SearchExpansion, SearchExplain, SearchGraphContext, + SearchPrefilter, SearchRecursive, Security, Service, Storage, TtlDays, }; fn test_ranking() -> Ranking { @@ -403,7 +403,7 @@ mod tests { update_sim_threshold: 0.8, candidate_k: 10, top_k: 5, - policy: MemoryPolicy::default(), + policy: MemoryPolicy { rules: vec![] }, }, search: Search { expansion: SearchExpansion { @@ -425,8 +425,18 @@ mod tests { candidate_retention_days: 2, write_mode: "outbox".to_string(), }, - recursive: Default::default(), - graph_context: Default::default(), + recursive: SearchRecursive { + enabled: false, + max_depth: 2, + max_children_per_node: 4, + max_nodes_per_scope: 32, + max_total_nodes: 256, + }, + graph_context: SearchGraphContext { + enabled: false, + max_facts_per_item: 16, + max_evidence_notes_per_fact: 16, + }, }, ranking: test_ranking(), lifecycle: Lifecycle { diff --git a/packages/elf-domain/tests/domain.rs b/packages/elf-domain/tests/domain.rs index b3e9c5d0..db3dfbc9 100644 --- a/packages/elf-domain/tests/domain.rs +++ b/packages/elf-domain/tests/domain.rs @@ -11,7 +11,8 @@ use elf_config::{ RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits, RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles, ScopePrecedence, ScopeWriteAllowed, Scopes, Search, SearchCache, SearchDynamic, - SearchExpansion, SearchExplain, SearchPrefilter, Security, Service, Storage, TtlDays, + SearchExpansion, SearchExplain, SearchGraphContext, SearchPrefilter, SearchRecursive, Security, + Service, Storage, TtlDays, }; use elf_domain::{evidence, ttl}; @@ -145,7 +146,7 @@ fn base_config() -> Config { update_sim_threshold: 0.85, candidate_k: 60, top_k: 12, - policy: MemoryPolicy::default(), + policy: MemoryPolicy { rules: vec![] }, }, search: Search { expansion: SearchExpansion { @@ -167,8 +168,18 @@ fn base_config() -> Config { candidate_retention_days: 2, write_mode: "outbox".to_string(), }, - recursive: Default::default(), - graph_context: Default::default(), + recursive: SearchRecursive { + enabled: false, + max_depth: 2, + max_children_per_node: 4, + max_nodes_per_scope: 32, + max_total_nodes: 256, + }, + graph_context: SearchGraphContext { + enabled: false, + max_facts_per_item: 16, + max_evidence_notes_per_fact: 16, + }, }, ranking: test_ranking(), lifecycle: Lifecycle { diff --git a/packages/elf-domain/tests/memory_policy.rs b/packages/elf-domain/tests/memory_policy.rs index 678d2c45..18261e00 100644 --- a/packages/elf-domain/tests/memory_policy.rs +++ b/packages/elf-domain/tests/memory_policy.rs @@ -10,7 +10,8 @@ use elf_config::{ RankingBlendSegment, RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits, RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles, ScopePrecedence, ScopeWriteAllowed, Scopes, Search, SearchCache, SearchDynamic, - SearchExpansion, SearchExplain, SearchPrefilter, Security, Service, Storage, TtlDays, + SearchExpansion, SearchExplain, SearchGraphContext, SearchPrefilter, SearchRecursive, Security, + Service, Storage, TtlDays, }; use elf_domain::memory_policy::{self, MemoryPolicyDecision, MemoryPolicyEvaluation}; @@ -186,8 +187,18 @@ fn memory_policy_search_config() -> Search { candidate_retention_days: 2, write_mode: "outbox".to_string(), }, - recursive: Default::default(), - graph_context: Default::default(), + recursive: SearchRecursive { + enabled: false, + max_depth: 2, + max_children_per_node: 4, + max_nodes_per_scope: 32, + max_total_nodes: 256, + }, + graph_context: SearchGraphContext { + enabled: false, + max_facts_per_item: 16, + max_evidence_notes_per_fact: 16, + }, } } diff --git a/packages/elf-service/tests/acceptance/chunk_search.rs b/packages/elf-service/tests/acceptance/chunk_search.rs index fddc5124..ec7eeb50 100644 --- a/packages/elf-service/tests/acceptance/chunk_search.rs +++ b/packages/elf-service/tests/acceptance/chunk_search.rs @@ -1044,12 +1044,10 @@ async fn search_details_payload_level_shapes_text_and_fields() { }; let note_id = Uuid::new_v4(); let chunk_id = Uuid::new_v4(); - let note_text = concat!( - "This is the long note body used for detail shaping. It contains enough tokens to show ", - "truncation and should be reduced for compact payload levels. The extra detail keeps ", - "running with repeated operational context about source references, structured fields, ", - "session hydration, ranking metadata, and payload contracts so l0 cannot equal the raw note.", - ); + let max_note_chars = context.service.cfg.memory.max_note_chars as usize; + let note_text_seed = + "This is the long note body used for detail shaping and payload truncation. "; + let note_text = note_text_seed.repeat((max_note_chars / note_text_seed.len()) + 2); let source_ref = serde_json::json!({ "schema": "note_source_ref/v1", "locator": { @@ -1060,12 +1058,13 @@ async fn search_details_payload_level_shapes_text_and_fields() { }); let structured_summary = "Structured summary about payload levels and compact text behavior."; let field_id = Uuid::new_v4(); - let max_note_chars = context.service.cfg.memory.max_note_chars as usize; + + assert!(note_text.len() > max_note_chars); insert_note_with_importance_and_source_ref( &context.service.db.pool, note_id, - note_text, + note_text.as_str(), &context.embedding_version, 0.8_f32, 1.0, @@ -1081,12 +1080,20 @@ async fn search_details_payload_level_shapes_text_and_fields() { 0, 0, note_text.len() as i32, - note_text, + note_text.as_str(), &context.embedding_version, ) .await; - upsert_point(&context.service, chunk_id, note_id, 0, 0, note_text.len() as i32, note_text) - .await; + upsert_point( + &context.service, + chunk_id, + note_id, + 0, + 0, + note_text.len() as i32, + note_text.as_str(), + ) + .await; let index = context .service @@ -1128,8 +1135,9 @@ async fn search_details_payload_level_shapes_text_and_fields() { ) .await; - assert!(l0.text.len() <= max_note_chars); - assert!(l1.text.len() <= max_note_chars); + assert!(l0.text.chars().count() <= max_note_chars + 3); + assert!(l1.text.chars().count() <= max_note_chars + 3); + assert!(l0.text.ends_with("...")); assert_eq!(l2.text, note_text); assert_ne!(l0.text, l1.text); assert_ne!(l0.text, note_text); diff --git a/packages/elf-service/tests/acceptance/suite.rs b/packages/elf-service/tests/acceptance/suite.rs index 97c28bdc..0d9839f4 100644 --- a/packages/elf-service/tests/acceptance/suite.rs +++ b/packages/elf-service/tests/acceptance/suite.rs @@ -35,12 +35,12 @@ use tokenizers::{Tokenizer, models::wordlevel::WordLevel}; use tokio::time; use elf_config::{ - Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, Postgres, - ProviderConfig, Ranking, RankingBlend, RankingBlendSegment, RankingDeterministic, + Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, MemoryPolicy, + Postgres, ProviderConfig, Ranking, RankingBlend, RankingBlendSegment, RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits, RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles, ScopePrecedence, ScopeWriteAllowed, - Scopes, Search, SearchCache, SearchDynamic, SearchExpansion, SearchExplain, SearchPrefilter, - Security, Service, Storage, TtlDays, + Scopes, Search, SearchCache, SearchDynamic, SearchExpansion, SearchExplain, SearchGraphContext, + SearchPrefilter, SearchRecursive, Security, Service, Storage, TtlDays, }; use elf_service::{ BoxFuture, ElfService, EmbeddingProvider, ExtractorProvider, RerankProvider, Result, @@ -200,31 +200,9 @@ pub fn test_config( update_sim_threshold: 0.85, candidate_k: 60, top_k: 12, - policy: Default::default(), - }, - search: Search { - expansion: SearchExpansion { - mode: "off".to_string(), - max_queries: 4, - include_original: true, - }, - dynamic: SearchDynamic { min_candidates: 10, min_top_score: 0.12 }, - prefilter: SearchPrefilter { max_candidates: 0 }, - cache: SearchCache { - enabled: true, - expansion_ttl_days: 7, - rerank_ttl_days: 7, - max_payload_bytes: Some(262_144), - }, - explain: SearchExplain { - retention_days: 7, - capture_candidates: false, - candidate_retention_days: 2, - write_mode: "outbox".to_string(), - }, - recursive: Default::default(), - graph_context: Default::default(), + policy: MemoryPolicy { rules: vec![] }, }, + search: test_search(), ranking: test_ranking(), lifecycle: Lifecycle { ttl_days: TtlDays { @@ -330,6 +308,42 @@ fn test_tokenizer_repo(collection: &str) -> String { tokenizer_path.to_string_lossy().into_owned() } +fn test_search() -> Search { + Search { + expansion: SearchExpansion { + mode: "off".to_string(), + max_queries: 4, + include_original: true, + }, + dynamic: SearchDynamic { min_candidates: 10, min_top_score: 0.12 }, + prefilter: SearchPrefilter { max_candidates: 0 }, + cache: SearchCache { + enabled: true, + expansion_ttl_days: 7, + rerank_ttl_days: 7, + max_payload_bytes: Some(262_144), + }, + explain: SearchExplain { + retention_days: 7, + capture_candidates: false, + candidate_retention_days: 2, + write_mode: "outbox".to_string(), + }, + recursive: SearchRecursive { + enabled: false, + max_depth: 2, + max_children_per_node: 4, + max_nodes_per_scope: 32, + max_total_nodes: 256, + }, + graph_context: SearchGraphContext { + enabled: false, + max_facts_per_item: 16, + max_evidence_notes_per_fact: 16, + }, + } +} + fn test_ranking() -> Ranking { Ranking { recency_tau_days: 60.0, diff --git a/packages/elf-service/tests/service.rs b/packages/elf-service/tests/service.rs index 3f624d89..7443e882 100644 --- a/packages/elf-service/tests/service.rs +++ b/packages/elf-service/tests/service.rs @@ -11,12 +11,13 @@ use serde_json::{Map, Value}; use sqlx::PgPool; use elf_config::{ - Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, Postgres, - ProviderConfig, Qdrant, Ranking, RankingBlend, RankingBlendSegment, RankingDeterministic, - RankingDeterministicDecay, RankingDeterministicHits, RankingDeterministicLexical, - RankingDiversity, RankingRetrievalSources, ReadProfiles, ScopePrecedence, ScopeWriteAllowed, - Scopes, Search, SearchCache, SearchDynamic, SearchExpansion, SearchExplain, SearchPrefilter, - Security, Service, Storage, TtlDays, + Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, MemoryPolicy, + Postgres, ProviderConfig, Qdrant, Ranking, RankingBlend, RankingBlendSegment, + RankingDeterministic, RankingDeterministicDecay, RankingDeterministicHits, + RankingDeterministicLexical, RankingDiversity, RankingRetrievalSources, ReadProfiles, + ScopePrecedence, ScopeWriteAllowed, Scopes, Search, SearchCache, SearchDynamic, + SearchExpansion, SearchExplain, SearchGraphContext, SearchPrefilter, SearchRecursive, Security, + Service, Storage, TtlDays, }; use elf_service::{ AddNoteInput, AddNoteRequest, BoxFuture, ElfService, EmbeddingProvider, Error, @@ -168,7 +169,7 @@ fn test_config() -> Config { update_sim_threshold: 0.8, candidate_k: 10, top_k: 5, - policy: Default::default(), + policy: MemoryPolicy { rules: vec![] }, }, search: Search { expansion: SearchExpansion { @@ -190,8 +191,18 @@ fn test_config() -> Config { candidate_retention_days: 2, write_mode: "outbox".to_string(), }, - recursive: Default::default(), - graph_context: Default::default(), + recursive: SearchRecursive { + enabled: false, + max_depth: 2, + max_children_per_node: 4, + max_nodes_per_scope: 32, + max_total_nodes: 256, + }, + graph_context: SearchGraphContext { + enabled: false, + max_facts_per_item: 16, + max_evidence_notes_per_fact: 16, + }, }, ranking: test_ranking(), lifecycle: Lifecycle { diff --git a/scripts/consolidation-harness.sh b/scripts/consolidation-harness.sh index b92a041e..e3ceddfa 100755 --- a/scripts/consolidation-harness.sh +++ b/scripts/consolidation-harness.sh @@ -145,9 +145,10 @@ dsn = "${PG_DSN}" pool_max_conns = 10 [storage.qdrant] -collection = "${QDRANT_COLLECTION}" -url = "${QDRANT_GRPC_URL}" -vector_dim = ${VECTOR_DIM_TOML} +collection = "${QDRANT_COLLECTION}" +docs_collection = "${QDRANT_COLLECTION}_docs" +url = "${QDRANT_GRPC_URL}" +vector_dim = ${VECTOR_DIM_TOML} [providers.embedding] api_base = "http://127.0.0.1" @@ -207,6 +208,12 @@ max_notes_per_add_event = 3 top_k = ${TOP_K} update_sim_threshold = 0.85 +[memory.policy] + +[[memory.policy.rules]] +min_confidence = 0.0 +min_importance = 0.0 + [chunking] enabled = true max_tokens = 512 @@ -236,6 +243,18 @@ capture_candidates = false candidate_retention_days = 2 write_mode = "outbox" +[search.recursive] +enabled = false +max_children_per_node = 4 +max_depth = 2 +max_nodes_per_scope = 32 +max_total_nodes = 256 + +[search.graph_context] +enabled = false +max_evidence_notes_per_fact = 16 +max_facts_per_item = 16 + [ranking] recency_tau_days = 60 tie_breaker_weight = 0.1 diff --git a/scripts/context-misranking-harness.sh b/scripts/context-misranking-harness.sh index 51508743..3290fdef 100755 --- a/scripts/context-misranking-harness.sh +++ b/scripts/context-misranking-harness.sh @@ -132,9 +132,10 @@ dsn = "${PG_DSN}" pool_max_conns = 10 [storage.qdrant] -collection = "${QDRANT_COLLECTION}" -url = "${QDRANT_GRPC_URL}" -vector_dim = ${VECTOR_DIM_TOML} +collection = "${QDRANT_COLLECTION}" +docs_collection = "${QDRANT_COLLECTION}_docs" +url = "${QDRANT_GRPC_URL}" +vector_dim = ${VECTOR_DIM_TOML} [providers.embedding] api_base = "http://127.0.0.1" @@ -194,6 +195,12 @@ max_notes_per_add_event = 3 top_k = 12 update_sim_threshold = 0.85 +[memory.policy] + +[[memory.policy.rules]] +min_confidence = 0.0 +min_importance = 0.0 + [chunking] enabled = true max_tokens = 512 @@ -223,6 +230,18 @@ capture_candidates = false candidate_retention_days = 2 write_mode = "outbox" +[search.recursive] +enabled = false +max_children_per_node = 4 +max_depth = 2 +max_nodes_per_scope = 32 +max_total_nodes = 256 + +[search.graph_context] +enabled = false +max_evidence_notes_per_fact = 16 +max_facts_per_item = 16 + [ranking] recency_tau_days = 60 tie_breaker_weight = 0.1 diff --git a/scripts/ranking-stability-harness.sh b/scripts/ranking-stability-harness.sh index cc55c367..fefb1a0d 100755 --- a/scripts/ranking-stability-harness.sh +++ b/scripts/ranking-stability-harness.sh @@ -121,9 +121,10 @@ dsn = "${PG_DSN}" pool_max_conns = 10 [storage.qdrant] -collection = "${QDRANT_COLLECTION}" -url = "${QDRANT_GRPC_URL}" -vector_dim = ${VECTOR_DIM_TOML} +collection = "${QDRANT_COLLECTION}" +docs_collection = "${QDRANT_COLLECTION}_docs" +url = "${QDRANT_GRPC_URL}" +vector_dim = ${VECTOR_DIM_TOML} [providers.embedding] api_base = "http://127.0.0.1" @@ -183,6 +184,12 @@ max_notes_per_add_event = 3 top_k = ${TOP_K} update_sim_threshold = 0.85 +[memory.policy] + +[[memory.policy.rules]] +min_confidence = 0.0 +min_importance = 0.0 + [chunking] enabled = true max_tokens = 512 @@ -212,6 +219,18 @@ capture_candidates = false candidate_retention_days = 2 write_mode = "outbox" +[search.recursive] +enabled = false +max_children_per_node = 4 +max_depth = 2 +max_nodes_per_scope = 32 +max_total_nodes = 256 + +[search.graph_context] +enabled = false +max_evidence_notes_per_fact = 16 +max_facts_per_item = 16 + [ranking] recency_tau_days = 0 tie_breaker_weight = 0.0